{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [EDA] Child Mind Institute -Detect Sleep Status- \nDetect sleep onset and wake from wrist-worn accelerometer data 😪😴\n---\n[Objective]\n- Check data & deepen understanding.  \n\n[Procedure]\n1. Import libraries\n2. Load data\n3. Merge train_event to train_series\n4. Visualization of Each Series_id","metadata":{}},{"cell_type":"markdown","source":"# 📚 1. Import Libraries","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\n# general\nimport os, gc, re, ast, sys, copy, json, math\nimport time, string, pickle, random, joblib, itertools\nfrom pathlib import Path\nfrom typing import List\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# matrix calc\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nimport polars as pl\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\n\n# visualization\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\n%matplotlib inline\nimport seaborn as sns\n\n# get date\nimport datetime\ndate = datetime.datetime.now().strftime(\"%Y%m%d\")\nprint(date)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-06T15:18:06.702055Z","iopub.execute_input":"2023-09-06T15:18:06.702349Z","iopub.status.idle":"2023-09-06T15:18:07.840933Z","shell.execute_reply.started":"2023-09-06T15:18:06.702327Z","shell.execute_reply":"2023-09-06T15:18:07.839719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nINPUT_DIR = f'../input/child-mind-institute-detect-sleep-states/'\nOUTPUT_DIR = f'../working/'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR, exist_ok=True)\n\nLOG_DIR = f'./log/'\nif not os.path.exists(LOG_DIR):\n    os.makedirs(LOG_DIR, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:07.843476Z","iopub.execute_input":"2023-09-06T15:18:07.844369Z","iopub.status.idle":"2023-09-06T15:18:07.851089Z","shell.execute_reply.started":"2023-09-06T15:18:07.844314Z","shell.execute_reply":"2023-09-06T15:18:07.849611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ================================\n# logger\n# ================================\nimport logging\nfrom logging import Formatter, StreamHandler, getLogger\n\n# loggerオブジェクトの宣言\nlogger = getLogger(__name__)\n# loggerのログレベル設定(ハンドラに渡すエラーメッセージのレベル)\nlogger.setLevel(logging.DEBUG)\n# handlerの生成\nfilehandler = logging.FileHandler((Path(LOG_DIR)/f\"{date}_log.txt\"))\nstream_handler = StreamHandler()\n# handlerのログレベル設定(ハンドラが出力するエラーメッセージのレベル)\nfilehandler.setLevel(logging.DEBUG)\nstream_handler.setLevel(logging.DEBUG)\n# ログ出力フォーマット設定\nhandler_format = Formatter(\"[%(asctime)s][%(name)s][%(levelname)s][%(message)s]\")\nfilehandler.setFormatter(handler_format)\nstream_handler.setFormatter(handler_format)\n# loggerにhandlerをセット\nlogger.addHandler(filehandler)\nlogger.addHandler(stream_handler)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:07.852250Z","iopub.execute_input":"2023-09-06T15:18:07.853463Z","iopub.status.idle":"2023-09-06T15:18:07.873009Z","shell.execute_reply.started":"2023-09-06T15:18:07.853427Z","shell.execute_reply":"2023-09-06T15:18:07.872121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utils","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed: int):\n    \"\"\"Set random seeds for reproducibility.\n\n    Args:\n        seed (int): Seed value.\n\n    Returns:\n        None\n    \"\"\"    \n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n#     torch.backends.cudnn.benchmark = True\n    \nseed_everything(seed=42)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:07.874330Z","iopub.execute_input":"2023-09-06T15:18:07.875333Z","iopub.status.idle":"2023-09-06T15:18:07.891106Z","shell.execute_reply.started":"2023-09-06T15:18:07.875300Z","shell.execute_reply":"2023-09-06T15:18:07.890274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_df(df:pd.DataFrame, num:int=3, tail:bool=False)-> None:\n    \"\"\"show shape and head of df \n\n    Args:\n        df (pd.DataFrame): data\n        num (int, optional): length of raws. Defaults to 3:int.\n        tail (bool, optional): if True show tail. Defaults to False:bool.\n    \"\"\"\n    print(df.shape)\n    display(df.head(num))\n    if tail: \n        display(df.tail(num))","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:07.893252Z","iopub.execute_input":"2023-09-06T15:18:07.893708Z","iopub.status.idle":"2023-09-06T15:18:07.904126Z","shell.execute_reply.started":"2023-09-06T15:18:07.893682Z","shell.execute_reply":"2023-09-06T15:18:07.903109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. 💿 Load Data","metadata":{}},{"cell_type":"markdown","source":"## 2.1. train_series.parquet","metadata":{}},{"cell_type":"code","source":"# ===========================\n# train_series.parquet\n# ===========================\ntrain = pl.read_parquet(Path(INPUT_DIR)/\"train_series.parquet\")\nprint(\"=\"*80); print(\"train_series.parquet\"); print(\"=\"*80); show_df(train)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:07.905180Z","iopub.execute_input":"2023-09-06T15:18:07.905443Z","iopub.status.idle":"2023-09-06T15:18:21.291380Z","shell.execute_reply.started":"2023-09-06T15:18:07.905409Z","shell.execute_reply":"2023-09-06T15:18:21.290534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- train_series.parquet - Continuous recording of accelerometer data.\n    - series_id - Unique identifier\n    - step - An integer timestep \n    - timestamp - A corresponding datetime \n    - anglez -  \n        As calculated and described by the GGIR package.  \n        z-angle is a metric derived from individual accelerometer components.\n        that is commonly used in sleep detection, \n        and refers to the angle of the arm relative to **the vertical axis of the body**\n    - enmo -  \n        As calculated and described by the GGIR package.  \n        ENMO is the Euclidean Norm Minus One of all accelerometer signals.\n        Negative values rounded to zero.  \n        While no standard measure of acceleration exists in this space,   \n        this is one of the several commonly computed features","metadata":{}},{"cell_type":"code","source":"# unique_series_id\nprint(\"=\"*80); print(\"unique_series_id\"); print(\"=\"*80); \nprint(len(train[\"series_id\"].unique()))","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:21.292298Z","iopub.execute_input":"2023-09-06T15:18:21.292543Z","iopub.status.idle":"2023-09-06T15:18:22.825605Z","shell.execute_reply.started":"2023-09-06T15:18:21.292522Z","shell.execute_reply":"2023-09-06T15:18:22.824410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic statistic \nprint(\"=\"*80); print(\"basic statistic\"); print(\"=\"*80); \ntrain.describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:22.827233Z","iopub.execute_input":"2023-09-06T15:18:22.827564Z","iopub.status.idle":"2023-09-06T15:18:30.328627Z","shell.execute_reply.started":"2023-09-06T15:18:22.827534Z","shell.execute_reply":"2023-09-06T15:18:30.328016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_count per unique_series_id\nprint(\"=\"*80); print(\"data_count\"); print(\"=\"*80); \ntrain.groupby(\"series_id\").agg(\n    pl.count(\"series_id\").suffix(f\"_count\"),\n).sort","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:30.329590Z","iopub.execute_input":"2023-09-06T15:18:30.329860Z","iopub.status.idle":"2023-09-06T15:18:32.159167Z","shell.execute_reply.started":"2023-09-06T15:18:30.329838Z","shell.execute_reply":"2023-09-06T15:18:32.158258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show histgram\nfig, axs = plt.subplots(1, 2, figsize=(12, 5))\ncols = [\"anglez\", \"enmo\"]\n\nfor ax, col in zip(axs, cols):\n    ax.hist(train[col], alpha=0.5)\n    ax.set_title(f'Histogram of {col}')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:32.160045Z","iopub.execute_input":"2023-09-06T15:18:32.160255Z","iopub.status.idle":"2023-09-06T15:18:35.032030Z","shell.execute_reply.started":"2023-09-06T15:18:32.160234Z","shell.execute_reply":"2023-09-06T15:18:35.031087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show histgram (enmo <= 2)\nplt.hist(train.filter(pl.col(\"enmo\")<=2)[\"enmo\"], alpha=0.5, bins = 100)\nplt.title(f'Histogram of enmo')","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:35.033301Z","iopub.execute_input":"2023-09-06T15:18:35.034042Z","iopub.status.idle":"2023-09-06T15:18:42.870555Z","shell.execute_reply.started":"2023-09-06T15:18:35.034010Z","shell.execute_reply":"2023-09-06T15:18:42.869774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show histgram (enmo <= 0.25)\nplt.hist(train.filter(pl.col(\"enmo\")<=0.25)[\"enmo\"], alpha=0.5, bins = 100)\nplt.title(f'Histogram of enmo')","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:42.871568Z","iopub.execute_input":"2023-09-06T15:18:42.872349Z","iopub.status.idle":"2023-09-06T15:18:50.156969Z","shell.execute_reply.started":"2023-09-06T15:18:42.872320Z","shell.execute_reply":"2023-09-06T15:18:50.155615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2. test_series.parquet ","metadata":{}},{"cell_type":"code","source":"# ===========================\n# test_series.parquet\n# ===========================\ntest = pl.read_parquet(Path(INPUT_DIR)/\"test_series.parquet\")\nprint(\"=\"*80); print(\"test_series.parquet\"); print(\"=\"*80); show_df(test)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.158358Z","iopub.execute_input":"2023-09-06T15:18:50.158646Z","iopub.status.idle":"2023-09-06T15:18:50.172908Z","shell.execute_reply.started":"2023-09-06T15:18:50.158621Z","shell.execute_reply":"2023-09-06T15:18:50.171313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic statistic \nprint(\"=\"*80); print(\"basic statistic\"); print(\"=\"*80); \ntest.describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.178904Z","iopub.execute_input":"2023-09-06T15:18:50.179263Z","iopub.status.idle":"2023-09-06T15:18:50.190272Z","shell.execute_reply.started":"2023-09-06T15:18:50.179233Z","shell.execute_reply":"2023-09-06T15:18:50.188862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique_series_id\nprint(\"=\"*80); print(\"unique_series_id\"); print(\"=\"*80); \nprint(len(test[\"series_id\"].unique()))","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.191700Z","iopub.execute_input":"2023-09-06T15:18:50.192554Z","iopub.status.idle":"2023-09-06T15:18:50.203604Z","shell.execute_reply.started":"2023-09-06T15:18:50.192527Z","shell.execute_reply":"2023-09-06T15:18:50.202739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_count per unique_series_id\nprint(\"=\"*80); print(\"data_count\"); print(\"=\"*80); \ntest.groupby(\"series_id\").agg(\n    pl.count(\"series_id\").suffix(f\"_count\"),\n).sort","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.207740Z","iopub.execute_input":"2023-09-06T15:18:50.208138Z","iopub.status.idle":"2023-09-06T15:18:50.221118Z","shell.execute_reply.started":"2023-09-06T15:18:50.208107Z","shell.execute_reply":"2023-09-06T15:18:50.220196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show histgram\nfig, axs = plt.subplots(1, 2, figsize=(12, 5))\ncols = [\"anglez\", \"enmo\"]\n\nfor ax, col in zip(axs, cols):\n    ax.hist(test[col], alpha=0.5)\n    ax.set_title(f'Histogram of {col}')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.222376Z","iopub.execute_input":"2023-09-06T15:18:50.222687Z","iopub.status.idle":"2023-09-06T15:18:50.673733Z","shell.execute_reply.started":"2023-09-06T15:18:50.222660Z","shell.execute_reply":"2023-09-06T15:18:50.672713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> test_series.csv is just reference data for dataset structure.","metadata":{}},{"cell_type":"markdown","source":"## 2.3. train_events.csv","metadata":{}},{"cell_type":"code","source":"# ===========================\n# train_events.csv\n# ===========================\nevents = pl.read_csv(Path(INPUT_DIR)/\"train_events.csv\")\nprint(\"=\"*80); print(\"train_events.csv\"); print(\"=\"*80); show_df(events)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.674827Z","iopub.execute_input":"2023-09-06T15:18:50.675086Z","iopub.status.idle":"2023-09-06T15:18:50.708820Z","shell.execute_reply.started":"2023-09-06T15:18:50.675064Z","shell.execute_reply":"2023-09-06T15:18:50.707416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- train_events.csv - Sleep logs for series in the training set recording onset and wake events.\n    - series_id -  \n        Unique identifier for each series of accelerometer data in train_series.parquet.\n    - night -  \n        An enumeration of potential onset / wakeup event pairs.  \n        At most one pair of events can occur for each night.\n    - event -  \n        The type of event, whether onset or wakeup.\n    - step and timestamp -  \n        The recorded time of occurence of the event in the accelerometer series.","metadata":{}},{"cell_type":"code","source":"# basic statistic \nprint(\"=\"*80); print(\"basic statistic\"); print(\"=\"*80); \nevents.describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.709911Z","iopub.execute_input":"2023-09-06T15:18:50.710190Z","iopub.status.idle":"2023-09-06T15:18:50.728172Z","shell.execute_reply.started":"2023-09-06T15:18:50.710163Z","shell.execute_reply":"2023-09-06T15:18:50.725999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique_series_id\nprint(\"=\"*80); print(\"unique_series_id\"); print(\"=\"*80); \nprint(len(events[\"series_id\"].unique()))","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.729730Z","iopub.execute_input":"2023-09-06T15:18:50.730420Z","iopub.status.idle":"2023-09-06T15:18:50.737530Z","shell.execute_reply.started":"2023-09-06T15:18:50.730389Z","shell.execute_reply":"2023-09-06T15:18:50.736727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> The number of unique_series_id is the same as train_series.parquet.","metadata":{}},{"cell_type":"code","source":"# unique_event\nprint(\"=\"*80); print(\"event\"); print(\"=\"*80); \nprint(*events[\"event\"].unique())","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.738654Z","iopub.execute_input":"2023-09-06T15:18:50.739280Z","iopub.status.idle":"2023-09-06T15:18:50.756173Z","shell.execute_reply.started":"2023-09-06T15:18:50.739254Z","shell.execute_reply":"2023-09-06T15:18:50.754990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show histgram\nfig, axs = plt.subplots(1, 2, figsize=(12, 5))\ncols = [\"night\", \"step\"]\n\nfor ax, col in zip(axs, cols):\n    ax.hist(events[col], alpha=0.5)\n    ax.set_title(f'Histogram of {col}')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:50.757494Z","iopub.execute_input":"2023-09-06T15:18:50.757870Z","iopub.status.idle":"2023-09-06T15:18:51.129954Z","shell.execute_reply.started":"2023-09-06T15:18:50.757839Z","shell.execute_reply":"2023-09-06T15:18:51.128936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.4. sumple_submission.csv","metadata":{}},{"cell_type":"code","source":"# ===========================\n# sumple_submission.csv\n# ===========================\ndf_sub = pl.read_csv(Path(INPUT_DIR)/\"sample_submission.csv\")\nprint(\"=\"*80); print(\"sample_submission.csv\"); print(\"=\"*80); show_df(df_sub)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:51.130784Z","iopub.execute_input":"2023-09-06T15:18:51.131019Z","iopub.status.idle":"2023-09-06T15:18:51.144115Z","shell.execute_reply.started":"2023-09-06T15:18:51.130998Z","shell.execute_reply":"2023-09-06T15:18:51.142787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. 🖇️ Merge train_event to train_series","metadata":{}},{"cell_type":"code","source":"# train = train.join(events, on=\"series_id\")\n# show_df(df)\n\n# -> our notebook tried to allocate more memory than is available. It has restarted.\n# Memory error","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:51.145298Z","iopub.execute_input":"2023-09-06T15:18:51.145565Z","iopub.status.idle":"2023-09-06T15:18:51.149831Z","shell.execute_reply.started":"2023-09-06T15:18:51.145544Z","shell.execute_reply":"2023-09-06T15:18:51.148997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del events\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:51.150980Z","iopub.execute_input":"2023-09-06T15:18:51.152055Z","iopub.status.idle":"2023-09-06T15:18:51.163970Z","shell.execute_reply.started":"2023-09-06T15:18:51.152007Z","shell.execute_reply":"2023-09-06T15:18:51.162988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. 🌌 Visualization of Each Series_id","metadata":{}},{"cell_type":"code","source":"# get_unique_id_list\nunique_id_list = list(train[\"series_id\"].unique())","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:51.165398Z","iopub.execute_input":"2023-09-06T15:18:51.165791Z","iopub.status.idle":"2023-09-06T15:18:52.638437Z","shell.execute_reply.started":"2023-09-06T15:18:51.165760Z","shell.execute_reply":"2023-09-06T15:18:52.637325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cast_before_merge\nevents = events.with_columns(\n    events.get_column(\"step\").cast(pl.UInt32).alias(\"step\")\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:52.640087Z","iopub.execute_input":"2023-09-06T15:18:52.640435Z","iopub.status.idle":"2023-09-06T15:18:52.647757Z","shell.execute_reply.started":"2023-09-06T15:18:52.640404Z","shell.execute_reply":"2023-09-06T15:18:52.647024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 1\n\n_train = train.filter(pl.col(\"series_id\")==unique_id_list[idx])\ndf = _train.join(events, how=\"left\", on=[\"series_id\", \"step\", \"timestamp\"])\nshow_df(df)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:52.648626Z","iopub.execute_input":"2023-09-06T15:18:52.648918Z","iopub.status.idle":"2023-09-06T15:18:53.142117Z","shell.execute_reply.started":"2023-09-06T15:18:52.648895Z","shell.execute_reply":"2023-09-06T15:18:53.141228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cast timestamp\ndf = df.with_columns(\n    df.get_column(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%z\").dt.replace_time_zone(None).alias(\"timestamp\")\n)\nshow_df(df)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:53.143372Z","iopub.execute_input":"2023-09-06T15:18:53.143666Z","iopub.status.idle":"2023-09-06T15:18:53.471592Z","shell.execute_reply.started":"2023-09-06T15:18:53.143640Z","shell.execute_reply":"2023-09-06T15:18:53.470972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get dummies\ndf = df.with_columns(\n    df.get_column(\"event\").to_dummies()\n)\nshow_df(df)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:53.472571Z","iopub.execute_input":"2023-09-06T15:18:53.473145Z","iopub.status.idle":"2023-09-06T15:18:53.501387Z","shell.execute_reply.started":"2023-09-06T15:18:53.473121Z","shell.execute_reply":"2023-09-06T15:18:53.500608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show timeseries data\nfig, axs = plt.subplots(4, 1, figsize=(20, 8))\nunuse_cols = [\"series_id\", \"step\", \"timestamp\", \"night\", \"event\", \"event_null\"]\ncols = [col for col in df.columns if col not in unuse_cols]\n\nfor ax, col in zip(axs, cols):\n    ax.plot(df[\"timestamp\"], df[col])\n    ax.set_ylabel(col)\n    \nax.set_xlabel(\"timestamp\")\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:53.502395Z","iopub.execute_input":"2023-09-06T15:18:53.503019Z","iopub.status.idle":"2023-09-06T15:18:54.685260Z","shell.execute_reply.started":"2023-09-06T15:18:53.502987Z","shell.execute_reply":"2023-09-06T15:18:54.684154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## define function","metadata":{}},{"cell_type":"code","source":"def show_ts(train:pl.DataFrame, events:pl.DataFrame, idx:int=0)->None:\n    \"\"\"show time series data\n    Args:\n        train(pl.DataFrame):train_series\n        events(pl.DataFrame):train_event\n        idx(int):number of unique series_id\n    Returns:\n        None\n    \"\"\"\n    _train = train.filter(pl.col(\"series_id\")==unique_id_list[idx])\n    # merge\n    df = _train.join(events, how=\"left\", on=[\"series_id\", \"step\", \"timestamp\"])\n    # cast datetime\n    df = df.with_columns(\n        df.get_column(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%z\").dt.replace_time_zone(None).alias(\"timestamp\"),\n    )\n    # get dummies\n    df = df.with_columns(\n        df.get_column(\"event\").to_dummies()\n    )\n\n    # show timeseries data\n    fig, axs = plt.subplots(4, 1, figsize=(20, 8))\n    unuse_cols = [\"series_id\", \"step\", \"timestamp\", \"night\", \"event\", \"event_null\"]\n    cols = [col for col in df.columns if col not in unuse_cols]\n\n    for ax, col in zip(axs, cols):\n        ax.plot(df[\"timestamp\"], df[col])\n        ax.set_ylabel(col)\n\n    ax.set_xlabel(\"timestamp\")\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:54.686678Z","iopub.execute_input":"2023-09-06T15:18:54.687088Z","iopub.status.idle":"2023-09-06T15:18:54.695614Z","shell.execute_reply.started":"2023-09-06T15:18:54.687029Z","shell.execute_reply":"2023-09-06T15:18:54.694220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1, 5):\n    print(\"=\"*80); print(f\"series_id: {unique_id_list[idx]}\"); print(\"=\"*80);\n    show_ts(train, events, idx)","metadata":{"execution":{"iopub.status.busy":"2023-09-06T15:18:54.697076Z","iopub.execute_input":"2023-09-06T15:18:54.697592Z","iopub.status.idle":"2023-09-06T15:19:02.331737Z","shell.execute_reply.started":"2023-09-06T15:18:54.697562Z","shell.execute_reply":"2023-09-06T15:19:02.330553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I wanna enjoy this competition 👍","metadata":{}}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}