{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Feature Engineering\n\nThis notebooks demonstrates a way to remove the noise from data using the principle component analysis.\n\nAuthor: Tolga Dincer","metadata":{}},{"cell_type":"code","source":"import gc\nfrom tqdm import tqdm\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.decomposition import PCA\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-11T20:05:30.900357Z","iopub.execute_input":"2023-10-11T20:05:30.901256Z","iopub.status.idle":"2023-10-11T20:05:30.906724Z","shell.execute_reply.started":"2023-10-11T20:05:30.901220Z","shell.execute_reply":"2023-10-11T20:05:30.905547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = Path(\"/kaggle/input/child-mind-institute-detect-sleep-states\")\n\nevents_file = \"train_events.csv\"\ntrain_series_file = root_dir / \"train_series.parquet\"\n\nevents = pd.read_csv(root_dir / events_file)","metadata":{"execution":{"iopub.status.busy":"2023-10-11T19:59:43.356697Z","iopub.execute_input":"2023-10-11T19:59:43.357912Z","iopub.status.idle":"2023-10-11T19:59:43.398786Z","shell.execute_reply.started":"2023-10-11T19:59:43.357876Z","shell.execute_reply":"2023-10-11T19:59:43.397516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(N):\n    RESOLUTION = 5\n    day_length = int(24 * 60 * 60 / RESOLUTION)\n\n    df = pd.read_parquet(train_series_file)\n\n    grouped = df.groupby([\"series_id\"])\n\n    filtered_groups = []\n    for _, group in tqdm(grouped):\n        trim_size = len(group) // day_length * day_length\n        trimmed_group = group[:trim_size:N]#[::N]\n        filtered_groups.append(trimmed_group)\n\n    df = pd.concat(filtered_groups)\n    df.reset_index(drop=True, inplace=True)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-11T20:00:41.940114Z","iopub.execute_input":"2023-10-11T20:00:41.940476Z","iopub.status.idle":"2023-10-11T20:00:41.946737Z","shell.execute_reply.started":"2023-10-11T20:00:41.940448Z","shell.execute_reply":"2023-10-11T20:00:41.945224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = load_data(N=12)\nsid_map = {sid: i for i, sid in enumerate(train.series_id.unique())}\nevents[\"series_id\"] = events[\"series_id\"].map(sid_map).astype(\"uint16\")\ntrain[\"series_id\"] = train[\"series_id\"].map(sid_map).astype(\"uint16\")","metadata":{"execution":{"iopub.status.busy":"2023-10-11T20:00:46.804643Z","iopub.execute_input":"2023-10-11T20:00:46.805022Z","iopub.status.idle":"2023-10-11T20:02:37.697646Z","shell.execute_reply.started":"2023-10-11T20:00:46.804985Z","shell.execute_reply":"2023-10-11T20:02:37.693094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_non_emptyids(events):\n    all_ids = events.series_id.unique()\n    empty_ids = events[events[\"step\"].isna()][\"series_id\"].unique()\n    non_empty_ids = set(all_ids) - set(empty_ids)\n    return non_empty_ids\n\ngood = list(find_non_emptyids(events))\ntrain = train[train.series_id.isin(good)]\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-11T20:03:15.452618Z","iopub.execute_input":"2023-10-11T20:03:15.453012Z","iopub.status.idle":"2023-10-11T20:03:34.016968Z","shell.execute_reply.started":"2023-10-11T20:03:15.452973Z","shell.execute_reply":"2023-10-11T20:03:34.015924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def analyze(series_id, length, feature):\n    y = train.query(f\"series_id=={series_id}\")[feature].values\n    y = y.reshape(-1, 30)\n\n    pca = PCA(n_components=2)\n    y_hat = pca.fit_transform(y)\n\n    fig, ax = plt.subplots(figsize=(10, 5), dpi=200)\n    ax.plot(y.flatten()[:length])\n    ax.plot(pca.inverse_transform(y_hat).flatten()[:length])\n    \n    plt.xlabel(\"Time (Minutes)\")\n    plt.ylabel(f\"{feature}\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-11T20:11:19.891256Z","iopub.execute_input":"2023-10-11T20:11:19.892599Z","iopub.status.idle":"2023-10-11T20:11:19.899388Z","shell.execute_reply.started":"2023-10-11T20:11:19.892559Z","shell.execute_reply":"2023-10-11T20:11:19.898201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series_id in train.series_id.unique():\n    analyze(series_id, length=1440, feature=\"enmo\")\n","metadata":{"execution":{"iopub.status.busy":"2023-10-11T20:17:15.038191Z","iopub.execute_input":"2023-10-11T20:17:15.038622Z","iopub.status.idle":"2023-10-11T20:17:29.886910Z","shell.execute_reply.started":"2023-10-11T20:17:15.038590Z","shell.execute_reply":"2023-10-11T20:17:29.885772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}