{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Browsing Features","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport polars as pl\n\npq_train_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n\ncsv_train_events = pl.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n\npq_test_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n\nprint(\"pq_train_series\\n\")\ndisplay(pq_train_series.schema)\nprint(\"\\ncsv_train_events\\n\")\ndisplay(csv_train_events.schema)\nprint(\"\\npq_test_series\\n\")\ndisplay(pq_test_series.schema)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-25T02:48:47.327065Z","iopub.execute_input":"2023-10-25T02:48:47.327854Z","iopub.status.idle":"2023-10-25T02:48:48.361236Z","shell.execute_reply.started":"2023-10-25T02:48:47.327814Z","shell.execute_reply":"2023-10-25T02:48:48.360026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"pq_train_series\\n\")\ndisplay(pq_train_series.select(pl.col(\"*\").is_null().sum()).collect())\nprint(\"\\ncsv_train_events\\n\")\ndisplay(csv_train_events.select(pl.col(\"*\").is_null().sum()))\nprint(\"\\npq_test_series\\n\")\ndisplay(pq_test_series.select(pl.col(\"*\").is_null().sum()).collect())","metadata":{"execution":{"iopub.status.busy":"2023-10-25T02:48:48.363387Z","iopub.execute_input":"2023-10-25T02:48:48.363974Z","iopub.status.idle":"2023-10-25T02:49:04.355061Z","shell.execute_reply.started":"2023-10-25T02:48:48.363936Z","shell.execute_reply":"2023-10-25T02:49:04.353855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Uniqe Series ID\nprint(\"pq_train_series\\n\")\ntrain_series_ids = pq_train_series.select(pl.col(\"series_id\")).unique().collect()\ndisplay(train_series_ids)\nprint(\"\\ncsv_train_events\\n\")\ntrain_event_series_ids = csv_train_events.select(pl.col(\"series_id\")).unique()\ndisplay(train_event_series_ids)\nprint(\"\\npq_test_series\\n\")\ntest_series_ids = pq_test_series.select(pl.col(\"series_id\")).unique().collect()\ndisplay(test_series_ids)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T02:49:04.356705Z","iopub.execute_input":"2023-10-25T02:49:04.357065Z","iopub.status.idle":"2023-10-25T02:49:15.069302Z","shell.execute_reply.started":"2023-10-25T02:49:04.357035Z","shell.execute_reply":"2023-10-25T02:49:15.068016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Count total record for each series_id\nfor train_series_id in train_series_ids.rows(named=True):\n    train_series_id = train_series_id[\"series_id\"]\n    sum_count = pq_train_series.select(pl.col(\"series_id\") == train_series_id).sum().collect()\n    print(f\"series_id:{train_series_id}, count:{sum_count.row(index=0)}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-25T02:49:15.072425Z","iopub.execute_input":"2023-10-25T02:49:15.072850Z","iopub.status.idle":"2023-10-25T03:06:32.600814Z","shell.execute_reply.started":"2023-10-25T02:49:15.072815Z","shell.execute_reply":"2023-10-25T03:06:32.599017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Count total record for each series_id\nfor test_series_id in test_series_ids.rows(named=True):\n    test_series_id = test_series_id[\"series_id\"]\n    sum_count = pq_test_series.select(pl.col(\"series_id\") == test_series_id).sum().collect()\n    print(f\"series_id:{test_series_id}, count:{sum_count.row(index=0)}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:06:32.604124Z","iopub.execute_input":"2023-10-25T03:06:32.604583Z","iopub.status.idle":"2023-10-25T03:06:32.619478Z","shell.execute_reply.started":"2023-10-25T03:06:32.604546Z","shell.execute_reply":"2023-10-25T03:06:32.618160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.1 Check train and test data contains the null series","metadata":{}},{"cell_type":"code","source":"print(\"csv_train_events null step series_id\\n\")\nnull_series_id = csv_train_events.select(pl.col(\"series_id\").where(pl.col(\"step\").is_null())).unique()\ndisplay(len(null_series_id))\ndisplay(null_series_id.head())","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:06:32.621542Z","iopub.execute_input":"2023-10-25T03:06:32.621978Z","iopub.status.idle":"2023-10-25T03:06:32.646269Z","shell.execute_reply.started":"2023-10-25T03:06:32.621944Z","shell.execute_reply":"2023-10-25T03:06:32.645129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"pq_train_series\\n\")\npq_train_series_id = pq_train_series.select(pl.col(\"series_id\")).unique().collect()\ndisplay(len(pq_train_series_id))\ndisplay(pq_train_series_id.head())\n\nprint(\"pq_train_series_id NOT match null series_id ? \\n\")\nnull_pq_train_series_id = pq_train_series_id.filter(~pl.col(\"series_id\").is_in(null_series_id[\"series_id\"].to_list()))\ndisplay(len(null_pq_train_series_id))\ndisplay(null_pq_train_series_id.head())","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:06:32.648139Z","iopub.execute_input":"2023-10-25T03:06:32.648505Z","iopub.status.idle":"2023-10-25T03:06:44.500618Z","shell.execute_reply.started":"2023-10-25T03:06:32.648476Z","shell.execute_reply":"2023-10-25T03:06:44.499467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"pq_test_series_id\\n\")\npq_test_series_id = pq_test_series.select(pl.col(\"series_id\")).unique().collect()\ndisplay(len(pq_test_series_id))\ndisplay(pq_test_series_id)\n\nprint(\"pq_test_series_id NOT match null series_id ? \\n\")\nnull_pq_test_series_id = pq_test_series_id.filter(~pl.col(\"series_id\").is_in(null_series_id[\"series_id\"].to_list()))\ndisplay(len(null_pq_test_series_id))\ndisplay(null_pq_test_series_id.head())","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:06:44.502607Z","iopub.execute_input":"2023-10-25T03:06:44.503099Z","iopub.status.idle":"2023-10-25T03:06:44.525944Z","shell.execute_reply.started":"2023-10-25T03:06:44.503047Z","shell.execute_reply":"2023-10-25T03:06:44.525089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.2 Summary train and test data","metadata":{}},{"cell_type":"code","source":"import datetime \n\ndt_transforms = [\n    pl.col('timestamp').str.to_datetime(), \n    (pl.col('timestamp').str.to_datetime().dt.year()-2000).cast(pl.UInt8).alias('year'), \n    pl.col('timestamp').str.to_datetime().dt.month().cast(pl.UInt8).alias('month'),\n    pl.col('timestamp').str.to_datetime().dt.day().cast(pl.UInt8).alias('day'), \n    pl.col('timestamp').str.to_datetime().dt.hour().cast(pl.UInt8).alias('hour'),\n]\n\nprint(\"pq_train_series\\n\")\ndisplay(pq_train_series.with_columns(dt_transforms).collect().describe())\nprint(\"\\ncsv_train_events\\n\")\ndisplay(csv_train_events.with_columns(dt_transforms).describe())\nprint(\"\\npq_test_series\\n\")\ndisplay(pq_test_series.with_columns(dt_transforms).collect().describe())","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:06:44.527130Z","iopub.execute_input":"2023-10-25T03:06:44.527505Z","iopub.status.idle":"2023-10-25T03:10:03.755429Z","shell.execute_reply.started":"2023-10-25T03:06:44.527473Z","shell.execute_reply":"2023-10-25T03:10:03.754313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"pq_train_series\\n\")\ndisplay(pq_train_series.with_columns(dt_transforms).collect().head())\nprint(\"\\ncsv_train_events\\n\")\ndisplay(csv_train_events.with_columns(dt_transforms).head())\nprint(\"\\npq_test_series\\n\")\ndisplay(pq_test_series.with_columns(dt_transforms).collect().head())","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:10:03.760298Z","iopub.execute_input":"2023-10-25T03:10:03.760725Z","iopub.status.idle":"2023-10-25T03:12:30.915953Z","shell.execute_reply.started":"2023-10-25T03:10:03.760690Z","shell.execute_reply":"2023-10-25T03:12:30.915047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.2 Summary train and test data (with null series_id removed)","metadata":{}},{"cell_type":"code","source":"null_series_id_list = null_series_id[\"series_id\"].to_list()\nprint(\"pq_train_series\\n\")\ndisplay(pq_train_series.with_columns(dt_transforms).\n        filter(~pl.col(\"series_id\").is_in(null_series_id_list)).\n               collect().describe())\nprint(\"\\ncsv_train_events\\n\")\ndisplay(csv_train_events.with_columns(dt_transforms).\n        filter(~pl.col(\"series_id\").is_in(null_series_id_list)).\n               describe())\nprint(\"\\npq_test_series\\n\")\ndisplay(pq_test_series.with_columns(dt_transforms).\n        filter(~pl.col(\"series_id\").is_in(null_series_id_list)).\n               collect().describe())","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:12:30.917356Z","iopub.execute_input":"2023-10-25T03:12:30.918282Z","iopub.status.idle":"2023-10-25T03:13:02.318760Z","shell.execute_reply.started":"2023-10-25T03:12:30.918250Z","shell.execute_reply":"2023-10-25T03:13:02.317777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Visualization","metadata":{}},{"cell_type":"markdown","source":"# 2.1 Histogram","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nplt.xlabel('pq_train_series step')\nplt.hist(pq_train_series.select(pl.col([\"step\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:02.320525Z","iopub.execute_input":"2023-10-25T03:13:02.321105Z","iopub.status.idle":"2023-10-25T03:13:02.722558Z","shell.execute_reply.started":"2023-10-25T03:13:02.321072Z","shell.execute_reply":"2023-10-25T03:13:02.721154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_train_series timestamp')\nplt.hist(pq_train_series.with_columns(dt_transforms)\n         .select(pl.col([\"timestamp\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:02.724378Z","iopub.execute_input":"2023-10-25T03:13:02.724816Z","iopub.status.idle":"2023-10-25T03:13:03.223213Z","shell.execute_reply.started":"2023-10-25T03:13:02.724780Z","shell.execute_reply":"2023-10-25T03:13:03.221779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_train_series anglez')\nplt.hist(pq_train_series.select(pl.col([\"anglez\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:03.225014Z","iopub.execute_input":"2023-10-25T03:13:03.225410Z","iopub.status.idle":"2023-10-25T03:13:03.578945Z","shell.execute_reply.started":"2023-10-25T03:13:03.225378Z","shell.execute_reply":"2023-10-25T03:13:03.577641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_train_series enmo')\nplt.hist(pq_train_series.select(pl.col([\"enmo\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:03.580896Z","iopub.execute_input":"2023-10-25T03:13:03.581267Z","iopub.status.idle":"2023-10-25T03:13:03.926476Z","shell.execute_reply.started":"2023-10-25T03:13:03.581235Z","shell.execute_reply":"2023-10-25T03:13:03.925065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_train_series year')\nplt.hist(pq_train_series.with_columns(dt_transforms)\n         .select(pl.col([\"year\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:03.928368Z","iopub.execute_input":"2023-10-25T03:13:03.928879Z","iopub.status.idle":"2023-10-25T03:13:04.355421Z","shell.execute_reply.started":"2023-10-25T03:13:03.928842Z","shell.execute_reply":"2023-10-25T03:13:04.354010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_train_series month')\nplt.hist(pq_train_series.with_columns(dt_transforms)\n         .select(pl.col([\"month\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:04.357378Z","iopub.execute_input":"2023-10-25T03:13:04.357847Z","iopub.status.idle":"2023-10-25T03:13:04.719815Z","shell.execute_reply.started":"2023-10-25T03:13:04.357805Z","shell.execute_reply":"2023-10-25T03:13:04.718488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_train_series day')\nplt.hist(pq_train_series.with_columns(dt_transforms)\n         .select(pl.col([\"day\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:04.721437Z","iopub.execute_input":"2023-10-25T03:13:04.721832Z","iopub.status.idle":"2023-10-25T03:13:05.158004Z","shell.execute_reply.started":"2023-10-25T03:13:04.721794Z","shell.execute_reply":"2023-10-25T03:13:05.156872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_train_series hour')\nplt.hist(pq_train_series.with_columns(dt_transforms)\n         .select(pl.col([\"hour\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:05.159487Z","iopub.execute_input":"2023-10-25T03:13:05.160201Z","iopub.status.idle":"2023-10-25T03:13:05.582734Z","shell.execute_reply.started":"2023-10-25T03:13:05.160167Z","shell.execute_reply":"2023-10-25T03:13:05.581771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('csv_train_events night')\nplt.hist(csv_train_events.with_columns(dt_transforms).select(pl.col([\"night\"])))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:05.584150Z","iopub.execute_input":"2023-10-25T03:13:05.584840Z","iopub.status.idle":"2023-10-25T03:13:05.920804Z","shell.execute_reply.started":"2023-10-25T03:13:05.584800Z","shell.execute_reply":"2023-10-25T03:13:05.919330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('csv_train_events event')\nplt.hist(csv_train_events.with_columns(dt_transforms).select(pl.col([\"event\"])))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:05.922379Z","iopub.execute_input":"2023-10-25T03:13:05.922807Z","iopub.status.idle":"2023-10-25T03:13:06.256173Z","shell.execute_reply.started":"2023-10-25T03:13:05.922765Z","shell.execute_reply":"2023-10-25T03:13:06.254864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('csv_train_events step')\nplt.hist(csv_train_events.with_columns(dt_transforms).select(pl.col([\"step\"])))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:06.258484Z","iopub.execute_input":"2023-10-25T03:13:06.259334Z","iopub.status.idle":"2023-10-25T03:13:06.801229Z","shell.execute_reply.started":"2023-10-25T03:13:06.259288Z","shell.execute_reply":"2023-10-25T03:13:06.799826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('csv_train_events timestamp')\nplt.hist(csv_train_events.with_columns(dt_transforms)\n         .select(pl.col([\"timestamp\"])))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:06.803374Z","iopub.execute_input":"2023-10-25T03:13:06.804335Z","iopub.status.idle":"2023-10-25T03:13:07.274960Z","shell.execute_reply.started":"2023-10-25T03:13:06.804285Z","shell.execute_reply":"2023-10-25T03:13:07.273737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('csv_train_events year')\nplt.hist(csv_train_events.with_columns(dt_transforms)\n         .select(pl.col([\"year\"])))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:07.276645Z","iopub.execute_input":"2023-10-25T03:13:07.277020Z","iopub.status.idle":"2023-10-25T03:13:07.648503Z","shell.execute_reply.started":"2023-10-25T03:13:07.276989Z","shell.execute_reply":"2023-10-25T03:13:07.647213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('csv_train_events month')\nplt.hist(csv_train_events.with_columns(dt_transforms)\n         .select(pl.col([\"month\"])))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:07.650080Z","iopub.execute_input":"2023-10-25T03:13:07.650486Z","iopub.status.idle":"2023-10-25T03:13:08.016435Z","shell.execute_reply.started":"2023-10-25T03:13:07.650452Z","shell.execute_reply":"2023-10-25T03:13:08.015081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('csv_train_events day')\nplt.hist(csv_train_events.with_columns(dt_transforms)\n         .select(pl.col([\"day\"])))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:08.018059Z","iopub.execute_input":"2023-10-25T03:13:08.018442Z","iopub.status.idle":"2023-10-25T03:13:08.365459Z","shell.execute_reply.started":"2023-10-25T03:13:08.018412Z","shell.execute_reply":"2023-10-25T03:13:08.364310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('csv_train_events hour')\nplt.hist(csv_train_events.with_columns(dt_transforms)\n         .select(pl.col([\"hour\"])))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:08.367005Z","iopub.execute_input":"2023-10-25T03:13:08.367950Z","iopub.status.idle":"2023-10-25T03:13:08.702567Z","shell.execute_reply.started":"2023-10-25T03:13:08.367915Z","shell.execute_reply":"2023-10-25T03:13:08.701181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_test_series step')\nplt.hist(pq_test_series.select(pl.col([\"step\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:08.711784Z","iopub.execute_input":"2023-10-25T03:13:08.712295Z","iopub.status.idle":"2023-10-25T03:13:09.054752Z","shell.execute_reply.started":"2023-10-25T03:13:08.712254Z","shell.execute_reply":"2023-10-25T03:13:09.053488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_train_series timestamp')\nplt.hist(pq_test_series.with_columns(dt_transforms)\n         .select(pl.col([\"timestamp\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:09.056838Z","iopub.execute_input":"2023-10-25T03:13:09.057318Z","iopub.status.idle":"2023-10-25T03:13:09.451505Z","shell.execute_reply.started":"2023-10-25T03:13:09.057275Z","shell.execute_reply":"2023-10-25T03:13:09.450227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_test_series anglez')\nplt.hist(pq_test_series.select(pl.col([\"anglez\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:09.453632Z","iopub.execute_input":"2023-10-25T03:13:09.454021Z","iopub.status.idle":"2023-10-25T03:13:09.785984Z","shell.execute_reply.started":"2023-10-25T03:13:09.453988Z","shell.execute_reply":"2023-10-25T03:13:09.784811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_test_series enmo')\nplt.hist(pq_test_series.select(pl.col([\"enmo\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:09.787404Z","iopub.execute_input":"2023-10-25T03:13:09.787752Z","iopub.status.idle":"2023-10-25T03:13:10.098886Z","shell.execute_reply.started":"2023-10-25T03:13:09.787724Z","shell.execute_reply":"2023-10-25T03:13:10.097579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_test_series year')\nplt.hist(pq_test_series.with_columns(dt_transforms)\n         .select(pl.col([\"year\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:10.100316Z","iopub.execute_input":"2023-10-25T03:13:10.100683Z","iopub.status.idle":"2023-10-25T03:13:10.405036Z","shell.execute_reply.started":"2023-10-25T03:13:10.100634Z","shell.execute_reply":"2023-10-25T03:13:10.403680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_test_series month')\nplt.hist(pq_test_series.with_columns(dt_transforms)\n         .select(pl.col([\"month\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:10.407266Z","iopub.execute_input":"2023-10-25T03:13:10.407777Z","iopub.status.idle":"2023-10-25T03:13:10.755302Z","shell.execute_reply.started":"2023-10-25T03:13:10.407731Z","shell.execute_reply":"2023-10-25T03:13:10.753868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_test_series day')\nplt.hist(pq_test_series.with_columns(dt_transforms)\n         .select(pl.col([\"day\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:10.757112Z","iopub.execute_input":"2023-10-25T03:13:10.757583Z","iopub.status.idle":"2023-10-25T03:13:11.096579Z","shell.execute_reply.started":"2023-10-25T03:13:10.757538Z","shell.execute_reply":"2023-10-25T03:13:11.095275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.xlabel('pq_test_series hour')\nplt.hist(pq_test_series.with_columns(dt_transforms)\n         .select(pl.col([\"hour\"])).fetch(n_rows=100_000))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:11.098205Z","iopub.execute_input":"2023-10-25T03:13:11.098587Z","iopub.status.idle":"2023-10-25T03:13:11.443248Z","shell.execute_reply.started":"2023-10-25T03:13:11.098554Z","shell.execute_reply":"2023-10-25T03:13:11.442100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.2 Scatter Plot","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_train_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_train_series.with_columns(dt_transforms)\n            .select(pl.col([\"timestamp\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:11.444749Z","iopub.execute_input":"2023-10-25T03:13:11.445128Z","iopub.status.idle":"2023-10-25T03:13:12.360016Z","shell.execute_reply.started":"2023-10-25T03:13:11.445095Z","shell.execute_reply":"2023-10-25T03:13:12.358760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_train_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_train_series.select(pl.col([\"anglez\"])).fetch(n_rows=100_000)\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:12.361934Z","iopub.execute_input":"2023-10-25T03:13:12.362327Z","iopub.status.idle":"2023-10-25T03:13:13.154392Z","shell.execute_reply.started":"2023-10-25T03:13:12.362294Z","shell.execute_reply":"2023-10-25T03:13:13.152964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_train_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_train_series.select(pl.col([\"enmo\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:13.156236Z","iopub.execute_input":"2023-10-25T03:13:13.156636Z","iopub.status.idle":"2023-10-25T03:13:13.927121Z","shell.execute_reply.started":"2023-10-25T03:13:13.156600Z","shell.execute_reply":"2023-10-25T03:13:13.925486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_train_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_train_series.with_columns(dt_transforms)\n             .select(pl.col([\"year\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:13.928854Z","iopub.execute_input":"2023-10-25T03:13:13.929214Z","iopub.status.idle":"2023-10-25T03:13:14.770107Z","shell.execute_reply.started":"2023-10-25T03:13:13.929184Z","shell.execute_reply":"2023-10-25T03:13:14.768746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_train_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_train_series.with_columns(dt_transforms)\n             .select(pl.col([\"month\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:14.771984Z","iopub.execute_input":"2023-10-25T03:13:14.772339Z","iopub.status.idle":"2023-10-25T03:13:15.792867Z","shell.execute_reply.started":"2023-10-25T03:13:14.772309Z","shell.execute_reply":"2023-10-25T03:13:15.791565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_train_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_train_series.with_columns(dt_transforms)\n             .select(pl.col([\"day\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:15.794440Z","iopub.execute_input":"2023-10-25T03:13:15.795389Z","iopub.status.idle":"2023-10-25T03:13:16.661995Z","shell.execute_reply.started":"2023-10-25T03:13:15.795355Z","shell.execute_reply":"2023-10-25T03:13:16.660764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_train_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_train_series.with_columns(dt_transforms)\n             .select(pl.col([\"hour\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:16.663769Z","iopub.execute_input":"2023-10-25T03:13:16.664120Z","iopub.status.idle":"2023-10-25T03:13:17.556408Z","shell.execute_reply.started":"2023-10-25T03:13:16.664091Z","shell.execute_reply":"2023-10-25T03:13:17.554629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_train_series.select(pl.col([\"anglez\"])).fetch(n_rows=100_000), \n            pq_train_series.select(pl.col([\"enmo\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:17.558397Z","iopub.execute_input":"2023-10-25T03:13:17.559635Z","iopub.status.idle":"2023-10-25T03:13:18.335348Z","shell.execute_reply.started":"2023-10-25T03:13:17.559583Z","shell.execute_reply":"2023-10-25T03:13:18.334074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(csv_train_events.with_columns(dt_transforms)[\"step\"], \n            csv_train_events.with_columns(dt_transforms)[\"event\"],\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:18.337246Z","iopub.execute_input":"2023-10-25T03:13:18.337625Z","iopub.status.idle":"2023-10-25T03:13:18.723959Z","shell.execute_reply.started":"2023-10-25T03:13:18.337593Z","shell.execute_reply":"2023-10-25T03:13:18.722288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(csv_train_events.with_columns(dt_transforms)[\"step\"], \n            csv_train_events.with_columns(dt_transforms)[\"night\"],\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:18.726248Z","iopub.execute_input":"2023-10-25T03:13:18.727275Z","iopub.status.idle":"2023-10-25T03:13:19.142718Z","shell.execute_reply.started":"2023-10-25T03:13:18.727233Z","shell.execute_reply":"2023-10-25T03:13:19.141744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(csv_train_events.with_columns(dt_transforms)[\"event\"], \n            csv_train_events.with_columns(dt_transforms)[\"night\"],\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:19.144204Z","iopub.execute_input":"2023-10-25T03:13:19.145425Z","iopub.status.idle":"2023-10-25T03:13:19.531820Z","shell.execute_reply.started":"2023-10-25T03:13:19.145378Z","shell.execute_reply":"2023-10-25T03:13:19.530442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(csv_train_events.with_columns(dt_transforms)[\"event\"], \n            csv_train_events.with_columns(dt_transforms)[\"year\"],\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:19.533610Z","iopub.execute_input":"2023-10-25T03:13:19.534120Z","iopub.status.idle":"2023-10-25T03:13:19.916500Z","shell.execute_reply.started":"2023-10-25T03:13:19.534076Z","shell.execute_reply":"2023-10-25T03:13:19.915224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(csv_train_events.with_columns(dt_transforms)[\"event\"], \n            csv_train_events.with_columns(dt_transforms)[\"month\"],\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:19.918747Z","iopub.execute_input":"2023-10-25T03:13:19.919606Z","iopub.status.idle":"2023-10-25T03:13:20.287818Z","shell.execute_reply.started":"2023-10-25T03:13:19.919545Z","shell.execute_reply":"2023-10-25T03:13:20.286014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(csv_train_events.with_columns(dt_transforms)[\"event\"], \n            csv_train_events.with_columns(dt_transforms)[\"day\"],\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:20.289943Z","iopub.execute_input":"2023-10-25T03:13:20.290457Z","iopub.status.idle":"2023-10-25T03:13:20.676093Z","shell.execute_reply.started":"2023-10-25T03:13:20.290411Z","shell.execute_reply":"2023-10-25T03:13:20.674618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(csv_train_events.with_columns(dt_transforms)[\"event\"], \n            csv_train_events.with_columns(dt_transforms)[\"hour\"],\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:20.677909Z","iopub.execute_input":"2023-10-25T03:13:20.678349Z","iopub.status.idle":"2023-10-25T03:13:21.040988Z","shell.execute_reply.started":"2023-10-25T03:13:20.678308Z","shell.execute_reply":"2023-10-25T03:13:21.039457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_test_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_test_series.with_columns(dt_transforms)\n            .select(pl.col([\"timestamp\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:21.042733Z","iopub.execute_input":"2023-10-25T03:13:21.043905Z","iopub.status.idle":"2023-10-25T03:13:21.442714Z","shell.execute_reply.started":"2023-10-25T03:13:21.043860Z","shell.execute_reply":"2023-10-25T03:13:21.441498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_test_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_test_series.select(pl.col([\"anglez\"])).fetch(n_rows=100_000)\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:21.444169Z","iopub.execute_input":"2023-10-25T03:13:21.444525Z","iopub.status.idle":"2023-10-25T03:13:21.793378Z","shell.execute_reply.started":"2023-10-25T03:13:21.444494Z","shell.execute_reply":"2023-10-25T03:13:21.792015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_test_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_test_series.select(pl.col([\"enmo\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:21.795418Z","iopub.execute_input":"2023-10-25T03:13:21.795852Z","iopub.status.idle":"2023-10-25T03:13:22.137111Z","shell.execute_reply.started":"2023-10-25T03:13:21.795816Z","shell.execute_reply":"2023-10-25T03:13:22.135820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_test_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_test_series.with_columns(dt_transforms)\n             .select(pl.col([\"year\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:22.139305Z","iopub.execute_input":"2023-10-25T03:13:22.139853Z","iopub.status.idle":"2023-10-25T03:13:22.484692Z","shell.execute_reply.started":"2023-10-25T03:13:22.139807Z","shell.execute_reply":"2023-10-25T03:13:22.483513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_test_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_test_series.with_columns(dt_transforms)\n             .select(pl.col([\"month\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:22.486579Z","iopub.execute_input":"2023-10-25T03:13:22.487018Z","iopub.status.idle":"2023-10-25T03:13:22.834432Z","shell.execute_reply.started":"2023-10-25T03:13:22.486983Z","shell.execute_reply":"2023-10-25T03:13:22.833013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_test_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_test_series.with_columns(dt_transforms)\n             .select(pl.col([\"day\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:22.836254Z","iopub.execute_input":"2023-10-25T03:13:22.837017Z","iopub.status.idle":"2023-10-25T03:13:23.167727Z","shell.execute_reply.started":"2023-10-25T03:13:22.836967Z","shell.execute_reply":"2023-10-25T03:13:23.166443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_test_series.select(pl.col([\"step\"])).fetch(n_rows=100_000), \n            pq_test_series.with_columns(dt_transforms)\n             .select(pl.col([\"hour\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:23.169901Z","iopub.execute_input":"2023-10-25T03:13:23.170267Z","iopub.status.idle":"2023-10-25T03:13:23.464161Z","shell.execute_reply.started":"2023-10-25T03:13:23.170237Z","shell.execute_reply":"2023-10-25T03:13:23.462915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nplt.scatter(pq_test_series.select(pl.col([\"anglez\"])).fetch(n_rows=100_000), \n            pq_test_series.select(pl.col([\"enmo\"])).fetch(n_rows=100_000),\n           )","metadata":{"execution":{"iopub.status.busy":"2023-10-25T03:13:23.465518Z","iopub.execute_input":"2023-10-25T03:13:23.466585Z","iopub.status.idle":"2023-10-25T03:13:23.805195Z","shell.execute_reply.started":"2023-10-25T03:13:23.466548Z","shell.execute_reply":"2023-10-25T03:13:23.803948Z"},"trusted":true},"execution_count":null,"outputs":[]}]}