{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-26T02:43:52.730856Z","iopub.execute_input":"2023-11-26T02:43:52.732563Z","iopub.status.idle":"2023-11-26T02:43:53.205407Z","shell.execute_reply.started":"2023-11-26T02:43:52.732459Z","shell.execute_reply":"2023-11-26T02:43:53.204383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport pyarrow.parquet as pq\nimport datetime as dt\nimport polars as pl\nfrom polars import col\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-11-26T02:43:57.186058Z","iopub.execute_input":"2023-11-26T02:43:57.186623Z","iopub.status.idle":"2023-11-26T02:44:12.975376Z","shell.execute_reply.started":"2023-11-26T02:43:57.186587Z","shell.execute_reply":"2023-11-26T02:44:12.974286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/child-mind-institute-detect-sleep-states/'\ndf_train = pl.read_parquet(path + \"train_series.parquet\")\ndf_events = pl.read_csv(path + 'train_events.csv')\ndf_test = pl.read_parquet(path + \"test_series.parquet\")\nprint(\"Uploading data success\")","metadata":{"execution":{"iopub.status.busy":"2023-11-26T02:44:15.389569Z","iopub.execute_input":"2023-11-26T02:44:15.391332Z","iopub.status.idle":"2023-11-26T02:44:38.014930Z","shell.execute_reply.started":"2023-11-26T02:44:15.391279Z","shell.execute_reply":"2023-11-26T02:44:38.013683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.with_columns(pl.col('timestamp').str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%z\").alias(\"timestamp\"))\ndf_train = df_train.with_columns(pl.col('anglez').cast(pl.Int16).alias(\"anglez\"))\ndf_train = df_train.with_columns(pl.col('enmo').cast(pl.UInt16).alias(\"enmo\"))\ndf_test = df_test.with_columns(pl.col('timestamp').str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%z\").alias(\"timestamp\"))\ndf_test = df_test.with_columns(pl.col('anglez').cast(pl.Int16).alias(\"anglez\"))\ndf_test = df_test.with_columns(pl.col('enmo').cast(pl.UInt16).alias(\"enmo\"))\ndf_events = df_events.with_columns(pl.col('timestamp').str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%z\").alias(\"timestamp\"))\nprint(\"Processing time success\")","metadata":{"execution":{"iopub.status.busy":"2023-11-26T02:44:43.300732Z","iopub.execute_input":"2023-11-26T02:44:43.301149Z","iopub.status.idle":"2023-11-26T02:45:31.616380Z","shell.execute_reply.started":"2023-11-26T02:44:43.301118Z","shell.execute_reply":"2023-11-26T02:45:31.615143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_ids_train = list(df_train['series_id'].unique(maintain_order=True))\nunique_ids_test = list(df_test['series_id'].unique(maintain_order=True))\nspl_train = {}\nspl_event = {}\nspl_test = {}\nfor name, data in df_train.group_by(\"series_id\"):\n    spl_train[name] = data\nfor name, data in df_test.group_by(\"series_id\"):\n    spl_test[name] = data\nfor name, data in df_events.group_by(\"series_id\"):\n    spl_event[name] = data\nprint(\"Spliting datas success\")","metadata":{"execution":{"iopub.status.busy":"2023-11-26T03:00:49.493310Z","iopub.execute_input":"2023-11-26T03:00:49.493782Z","iopub.status.idle":"2023-11-26T03:01:08.337212Z","shell.execute_reply.started":"2023-11-26T03:00:49.493749Z","shell.execute_reply":"2023-11-26T03:01:08.335490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Processing train_set\ntolerance=0\nfor i in unique_ids_train:\n  value = [1] * spl_train[i].shape[0]\n  for idx, step in enumerate(spl_event[i]['step']):\n    if step != None and spl_event[i]['event'][idx] == \"wakeup\":\n      start = spl_event[i]['step'][idx-1] if spl_event[i]['step'][idx-1]!=None else step-360\n      end = step\n      for x in range(start, end):\n        value[x] = 0\n  data = pl.Series('wake', value)\n  spl_train[i] = spl_train[i].with_columns(data.alias('wake'))\nprint(\"Processing training set success\")","metadata":{"execution":{"iopub.status.busy":"2023-11-26T03:05:11.555123Z","iopub.execute_input":"2023-11-26T03:05:11.555550Z","iopub.status.idle":"2023-11-26T03:05:26.791840Z","shell.execute_reply.started":"2023-11-26T03:05:11.555519Z","shell.execute_reply":"2023-11-26T03:05:26.790232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import catboost\n\nfeature = set(['anglez','enmo', 'anglez_5min_tot_var', 'anglez_30min_tot_var', 'anglez_120min_tot_var', 'anglez_240min_tot_var'])\nfor i in range(0,360,12):\n    name_anglez = 'angle_lag' + str(i)\n    feature.add(name_anglez)\n\ndef Data_process(data_):\n  lagging = 360\n\n  for i in range(0,lagging,12):\n    name_anglez = 'angle_lag' + str(i)\n    data_ = data_.with_columns(data_['anglez'].shift(i).alias(name_anglez))\n\n  data_ = data_.with_columns(pl.col('anglez').diff().abs().rolling_mean(60, center=True, min_periods=1).abs().cast(pl.UInt32).alias('anglez_5min_tot_var'))\n  data_ = data_.with_columns(pl.col('anglez').diff().abs().rolling_mean(360, center=True, min_periods=1).abs().cast(pl.UInt32).alias('anglez_30min_tot_var'))\n  data_ = data_.with_columns(pl.col('anglez').diff().abs().rolling_mean(360*4, center=True, min_periods=1).abs().cast(pl.UInt32).alias('anglez_120min_tot_var'))\n  data_ = data_.with_columns(pl.col('anglez').diff().abs().rolling_mean(360*8, center=True, min_periods=1).abs().cast(pl.UInt32).alias('anglez_240min_tot_var'))\n\n  return data_.to_pandas()\n\nprint(\"Making functions and feature set success\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train\ndel df_events\ndel df_test\nprint(\"Removing unnecessary dataframe success\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature = list(feature)\n\nCBoost = catboost.CatBoostClassifier(iterations=20, learning_rate=0.5, depth=8, loss_function='Logloss')\n\nfor i, id in enumerate(unique_ids_train):\n  print(i)\n  data = spl_train[id]\n  X = Data_process(data)\n  y = list(data['wake'])\n  train = catboost.Pool(data=X[feature], label=y)\n  if i == 0:\n    CBoost.fit(train)\n  else:\n    if len(set(y)) > 1:\n      CBoost.fit(train, init_model='model.cbm')\n  CBoost.save_model('model.cbm')\nprint(\"Training model success\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importances = CBoost.feature_importances_\nfeature_importances = pd.Series(importances, index=feature).sort_values()\nplt.figure(figsize=(10, 12))\nplt.barh(feature_importances.index, feature_importances.values)\nprint(\"Feature importance success\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = CBoost\n\nseries_ids = unique_ids_test\nevents = pl.DataFrame(schema={'series_id':str, 'step':int, 'event':str, 'score':float})\n\nfor id_ in series_ids:\n    print(id_)\n    series = pl.DataFrame(Data_process(spl_test[id_]))\n    del spl_test[id_]\n    scale_cols = [col for col in feature if series[col].std() !=0]\n    X = series.select(['series_id','step','timestamp'] + feature)\n\n    preds, probs = classifier.predict(X[feature].to_pandas()), classifier.predict_proba(X[feature].to_pandas())[:, 1]\n    print(preds)\n    X = X.with_columns(\n        pl.lit(preds.flatten()).cast(pl.Int8).alias('prediction'),\n        pl.lit(probs).alias('probability')\n                      )\n\n    pred_onsets = X.filter(X['prediction'].diff() > 0)['step'].to_list()\n    pred_wakeups = X.filter(X['prediction'].diff() < 0)['step'].to_list()\n\n    if len(pred_onsets) > 0 and len(pred_wakeups) > 0:\n        \n        if min(pred_wakeups) < min(pred_onsets) :\n            pred_wakeups = pred_wakeups[1:]\n        \n        if max(pred_onsets) > max(pred_wakeups) :\n            pred_onsets = pred_onsets[:-1]\n        sleep_periods = [(onset, wakeup) for onset, wakeup in zip(pred_onsets, pred_wakeups) if wakeup - onset >= 12 * 30]\n        print(sleep_periods)\n\n        for onset, wakeup in sleep_periods :\n            score = X.filter((pl.col('step') >= onset) & (pl.col('step') <= wakeup))['probability'].mean()\n            print(score)\n            events = events.vstack(pl.DataFrame().with_columns(\n                pl.Series([idx, idx]).alias('series_id'),\n                pl.Series([onset, wakeup]).alias('step'),\n                pl.Series(['onset', 'wakeup']).alias('event'),\n                pl.Series([score, score]).alias('score')\n            ))\n    del X\n    \n\nevents = events.to_pandas().reset_index().rename(columns={'index':'row_id'})\nprint(events)\nevents.to_csv('submission.csv', index=False)\nprint(\"Final success\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for root, dirs, files in os.walk(\"/kaggle/working/catboost_info/\", topdown=False):\n    for name in files:\n        os.remove(os.path.join(root, name))\n    for name in dirs:\n        os.rmdir(os.path.join(root, name))\nos.rmdir(\"/kaggle/working/catboost_info/\")\nos.remove(\"/kaggle/working/model.cbm\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}