{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-09-12T10:37:40.985942Z","iopub.execute_input":"2023-09-12T10:37:40.986377Z","iopub.status.idle":"2023-09-12T10:37:41.421832Z","shell.execute_reply.started":"2023-09-12T10:37:40.986344Z","shell.execute_reply":"2023-09-12T10:37:41.420874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We convert features to int8 and float16 type to conserve memory. (https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/438394)","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet\")\ntest  = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-12T10:42:26.974438Z","iopub.execute_input":"2023-09-12T10:42:26.974855Z","iopub.status.idle":"2023-09-12T10:42:34.270247Z","shell.execute_reply.started":"2023-09-12T10:42:26.974821Z","shell.execute_reply":"2023-09-12T10:42:34.269303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_features(df):\n    # parse the timestamp and create an \"hour\" feature\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"],utc=True)\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"minute\"] = df[\"timestamp\"].dt.minute\n    # feature cross\n    df[\"anglez_times_enmo\"] = abs(df[\"anglez\"]) * df[\"enmo\"]\n    # \"rolling\" features\n    periods = 50\n    df[\"anglez_diff\"] = df.groupby('series_id')['anglez'].diff(periods=periods).fillna(method=\"bfill\")\n    df[\"enmo_diff\"]   = df.groupby('series_id')['enmo'].diff(periods=periods).fillna(method=\"bfill\")\n    df[\"anglez_rolling\"] = df[\"anglez\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    df[\"enmo_rolling\"]   = df[\"enmo\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    df[\"anglez_diff_rolling\"] = df[\"anglez_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    df[\"enmo_diff_rolling\"]   = df[\"enmo_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    df[\"anglez_change_rate\"] = df[\"anglez\"].diff() / df[\"timestamp\"].diff().dt.total_seconds()\n    df[\"enmo_change_rate\"] = df[\"enmo\"].diff() / df[\"timestamp\"].diff().dt.total_seconds()\n    df['hour_sin'] = np.sin(2 * np.pi * df['hour'] / 24)\n    df['hour_cos'] = np.cos(2 * np.pi * df['hour'] / 24)\n\n    \n    \n    return df\n\n\ntrain = create_features(train)\ntest = create_features(test)","metadata":{"execution":{"iopub.status.busy":"2023-09-12T10:42:34.271971Z","iopub.execute_input":"2023-09-12T10:42:34.272285Z","iopub.status.idle":"2023-09-12T10:44:32.199708Z","shell.execute_reply.started":"2023-09-12T10:42:34.272258Z","shell.execute_reply":"2023-09-12T10:44:32.198427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Specify the columns and their new data types\ndtype_dict = {\n    'hour': 'int8',\n    'minute': 'int8',\n    'anglez': 'float16',\n    'enmo': 'float16',\n    \"anglez_times_enmo\": \"float16\",\n    \"anglez_diff\": \"float16\",\n    \"enmo_diff\": \"float16\",\n    \"anglez_rolling\": \"float16\",\n    \"enmo_rolling\": \"float16\",\n    \"anglez_diff_rolling\": \"float16\",\n    \"enmo_diff_rolling\": \"float16\",\n    \"anglez_change_rate\": \"float16\",\n    \"enmo_change_rate\": \"float16\",\n    \"hour_sin\": \"float16\",\n    \"hour_cos\": \"float16\"\n    # add other columns here\n}\n\n# Apply the data type conversion\ntrain = train.astype(dtype_dict)\ntest = test.astype(dtype_dict)","metadata":{"execution":{"iopub.status.busy":"2023-09-12T10:44:36.973934Z","iopub.execute_input":"2023-09-12T10:44:36.974412Z","iopub.status.idle":"2023-09-12T10:44:39.119871Z","shell.execute_reply.started":"2023-09-12T10:44:36.974365Z","shell.execute_reply":"2023-09-12T10:44:39.118848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-09-12T10:44:46.863371Z","iopub.execute_input":"2023-09-12T10:44:46.863921Z","iopub.status.idle":"2023-09-12T10:44:46.874012Z","shell.execute_reply.started":"2023-09-12T10:44:46.863872Z","shell.execute_reply":"2023-09-12T10:44:46.872763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}