{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pandas.api.types import is_datetime64_ns_dtype\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-23T16:16:40.968040Z","iopub.execute_input":"2023-09-23T16:16:40.968675Z","iopub.status.idle":"2023-09-23T16:16:41.431228Z","shell.execute_reply.started":"2023-09-23T16:16:40.968640Z","shell.execute_reply":"2023-09-23T16:16:41.429154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I used the the training data from the reduced dataset [\"Zzzs: Lightweight training dataset + target\"](https://www.kaggle.com/datasets/carlmcbrideellis/zzzs-lightweight-training-dataset-target)\n\n### Feature Engineering\n\nFeature engineering is the backbone of effective AI models. It involves crafting meaningful input variables that enable models to learn and make accurate predictions. Well-engineered features extract crucial information from raw data, enhancing model performance and interpretability. In AI, the saying \"garbage in, garbage out\" holds true; thoughtful feature engineering transforms data into valuable insights, making it an indispensable step in building robust and effective models. However, it's important to note that feature engineering is a highly context-dependent process.\n\n\nThis notebook performs feature engineering by adding various derived features. Here's a description of some ideas:\n\n-**Timestamp Derivatives**: It extracts time-related information from the 'timestamp' column, including hour, weekday, whether it's a weekend, month, and day.\n\n-**Feature Cross**: It creates a new feature by multiplying 'anglez' and 'enmo' columns.\n\n-**Difference**: For the 'enmo' and 'anglez' columns, it calculates difference for different time periods.\n\n-**Rolling Statistics**: For the 'enmo' and 'anglez' columns, it calculates various rolling statistics like mean, median, max, min, skewness, and kurtosis for different time periods.\n\n-**Time Lag and Lead**: It creates lag and lead features for 'enmo' and 'anglez' columns, allowing the model to consider past and future values.\n\nThis notebook serves as a template, demonstrating some feature engineering techniques that can be adapted and extended. The data with this feaure engineering can e download [here](https://www.kaggle.com/datasets/renatoreggiani/zzzs-training-dataset-target-feat-eng)","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet('/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-09-23T16:16:41.435357Z","iopub.execute_input":"2023-09-23T16:16:41.435927Z","iopub.status.idle":"2023-09-23T16:16:55.590659Z","shell.execute_reply.started":"2023-09-23T16:16:41.435878Z","shell.execute_reply":"2023-09-23T16:16:55.589416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n    df['series_id'] = df['series_id'].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    \n    return df","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-09-23T16:16:55.592154Z","iopub.execute_input":"2023-09-23T16:16:55.592554Z","iopub.status.idle":"2023-09-23T16:16:55.611822Z","shell.execute_reply.started":"2023-09-23T16:16:55.592514Z","shell.execute_reply":"2023-09-23T16:16:55.610096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal_awake = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.208 * np.pi) ** 24))\nsignal_onset = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.555 * np.pi) ** 24))\n\n\ndef feat_eng(df):\n    \n    print('Start Feat Eng')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df.sort_values(['series_id', 'timestamp'], inplace=True)\n    \n    df['hour'] = df['timestamp'].dt.hour.astype(np.int8)\n    \n    if df['hour'].nunique()==24:\n        df = pd.concat([df, pd.get_dummies(df['hour'], dtype=np.int8, prefix='hour', drop_first=True)], axis=1)\n    else:\n        for h in range(1, 24):\n            in_hour = 1 if h in df['hour'].unique() else 0\n            df[f'hour_{h}'] = in_hour\n    \n    df['month'] = df['timestamp'].dt.month.astype(np.int8)\n    \n    df['signal_onset'] = (df.timestamp.dt.hour * 60 + df.timestamp.dt.minute).map(signal_onset).astype(np.float32)\n    df['signal_awake'] = (df.timestamp.dt.hour * 60 + df.timestamp.dt.minute).map(signal_awake).astype(np.float32)\n    df[\"anglez\"] = abs(df[\"anglez\"])\n    \n    print('Created timestamp derivates')\n    gc.collect()\n    \n    # feature cross\n    df['anglez_x_enmo'] = df['anglez'] * df['enmo']\n    df['anglezabs_x_enmo'] = abs(df['anglez']) * df['enmo']\n    \n    \n    z = np.maximum(0., df.enmo - 0.02)\n    z = z.to_frame(name='lids').assign(series_id=df.series_id)\n    z = z.groupby('series_id')['lids'].rolling(120, center=True).agg('sum')\n    z = 100 / (z + 1)\n    z = z.to_frame().reset_index()\n    z = z.groupby('series_id')['lids'].rolling(360, center=True).agg('mean')\n    df['lids'] = z.astype(np.float32).values\n    \n    for col in ['enmo', 'anglez', 'anglez_x_enmo']:\n        for n in [21, 61, 121]:\n            \n            for agg in ['median', 'mean', 'max', 'min', 'std']:\n            \n                df[f'{col}_{agg}_{n}'] = df.groupby('series_id')[col].rolling(n, center=True).agg(agg).astype(np.float32).values\n                gc.collect()\n            \n            df[f'{col}_diff_{n}'] = df.groupby('series_id')[col].diff(periods=n).astype(np.float32)\n            df[f'{col}_emavg_{n}'] = df[col].ewm(span=n).mean().astype(np.float32)\n            \n            \n            df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs()\n            df[f'{col}_mad_{n}'] = df.groupby('series_id')[f'{col}_mad_{n}'].rolling(n, center=True).agg('median').astype(np.float32).values\n            \n#             df[f'{col}_skew_{n}'] = df.groupby('series_id')[col].rolling(n).skew().reset_index(drop=True).astype(np.float32)\n#             df[f'{col}_kurt_{n}'] = df.groupby('series_id')[col].rolling(n).kurt().reset_index(drop=True).astype(np.float32)\n            \n            df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}'].astype(np.float32)\n            \n            df[f'{col}_diff_{n}_mean'] = df[f'{col}_diff_{n}'].rolling(n,center=True).mean().astype(np.float32)\n            df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].rolling(n,center=True).max().astype(np.float32)\n    \n            gc.collect()\n            \n            \n        print(f'Created diff and smoothed derivates from {col}')\n\n    gc.collect() \n\n\n    df.bfill(inplace=True, limit=15)\n    df.ffill(inplace=True, limit=15)\n    df.dropna(inplace=True)\n\n    df = reduce_mem_usage(df)\n\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2023-09-23T16:16:55.614460Z","iopub.execute_input":"2023-09-23T16:16:55.614782Z","iopub.status.idle":"2023-09-23T16:16:55.643147Z","shell.execute_reply.started":"2023-09-23T16:16:55.614751Z","shell.execute_reply":"2023-09-23T16:16:55.642203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain['awake'] = train['awake'].astype(np.int8)\ntrain = feat_eng(train)","metadata":{"execution":{"iopub.status.busy":"2023-09-23T16:16:55.644248Z","iopub.execute_input":"2023-09-23T16:16:55.645019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_parquet('train_plus_feat.parquet', compression='gzip')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntest['timestamp'] = pd.to_datetime(test['timestamp'],utc=True)\ntest = feat_eng(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['awake'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}