{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-18T07:03:45.531105Z","iopub.execute_input":"2023-11-18T07:03:45.531553Z","iopub.status.idle":"2023-11-18T07:03:46.083632Z","shell.execute_reply.started":"2023-11-18T07:03:45.531518Z","shell.execute_reply":"2023-11-18T07:03:46.081856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\npd.set_option('display.max_columns',None)\npd.set_option('display.max_colwidth',None)\nimport seaborn as sns\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport matplotlib.dates as mdates\n!pip install pyspark\nfrom pyspark.sql import SparkSession\nfrom pyspark.sql.functions import *\nspark = SparkSession.builder.getOrCreate()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:03:46.086643Z","iopub.execute_input":"2023-11-18T07:03:46.087705Z","iopub.status.idle":"2023-11-18T07:04:51.893774Z","shell.execute_reply.started":"2023-11-18T07:03:46.087665Z","shell.execute_reply":"2023-11-18T07:04:51.892158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=spark.read.parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet',\\\n                       header=True, inferschema=True)\ndata.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:04:51.899221Z","iopub.execute_input":"2023-11-18T07:04:51.899745Z","iopub.status.idle":"2023-11-18T07:05:00.876894Z","shell.execute_reply.started":"2023-11-18T07:04:51.899699Z","shell.execute_reply":"2023-11-18T07:05:00.875890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''start_mem = data.memory_usage().sum() / 1024**2\nprint('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\nfor col in data.columns:\n    col_type = data[col].dtype    \n    if col_type != object:\n        c_min = data[col].min()\n        c_max = data[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                data[col] = data[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                data[col] = data[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                data[col] = data[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                data[col] = data[col].astype(np.int64)  \n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                data[col] = data[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                data[col] = data[col].astype(np.float32)\n            else:\n                data[col] = data[col].astype(np.float64)\n    else:\n        data[col] = data[col].astype('category')\n\nend_mem = data.memory_usage().sum() / 1024**2\nprint('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\nprint('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))'''","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:05:00.880465Z","iopub.execute_input":"2023-11-18T07:05:00.880961Z","iopub.status.idle":"2023-11-18T07:05:00.898763Z","shell.execute_reply.started":"2023-11-18T07:05:00.880918Z","shell.execute_reply":"2023-11-18T07:05:00.897782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=data.withColumn('timestamp',to_timestamp('timestamp'))\ndata=data.filter((data.timestamp.isNotNull())&(data.anglez.isNotNull())&(data.enmo.isNotNull()))\ndata.printSchema()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:05:00.900434Z","iopub.execute_input":"2023-11-18T07:05:00.902106Z","iopub.status.idle":"2023-11-18T07:05:01.054725Z","shell.execute_reply.started":"2023-11-18T07:05:00.902073Z","shell.execute_reply":"2023-11-18T07:05:01.053759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events=spark.read.csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv',\\\n                      inferSchema=True,header=True)\nevents=events.filter((events.timestamp.isNotNull())&(events.event.isNotNull()))\nevents.printSchema()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:05:01.055941Z","iopub.execute_input":"2023-11-18T07:05:01.056389Z","iopub.status.idle":"2023-11-18T07:05:03.104839Z","shell.execute_reply.started":"2023-11-18T07:05:01.056349Z","shell.execute_reply":"2023-11-18T07:05:03.103674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=data.select(concat_ws('_',data.series_id,data.timestamp).alias('key'),\\\n            'series_id', 'step', 'timestamp', 'anglez', 'enmo')\nevents=events.select(concat_ws('_',events.series_id,events.timestamp).alias('key'),\\\n                    'event')\ndata=data.join(events,on='key',how='left')\ndel events\ndata=data.drop('key')","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:05:03.106195Z","iopub.execute_input":"2023-11-18T07:05:03.106670Z","iopub.status.idle":"2023-11-18T07:05:03.350303Z","shell.execute_reply.started":"2023-11-18T07:05:03.106626Z","shell.execute_reply":"2023-11-18T07:05:03.349084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@udf\ndef map_events(ev):\n    if ev=='onset':\n        return 0\n    if ev=='wakeup':\n        return 1\n    else:\n        return -1\ndata=data.withColumn('event',map_events(col('event')).cast('int'))\ndata.printSchema()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:05:03.351696Z","iopub.execute_input":"2023-11-18T07:05:03.352155Z","iopub.status.idle":"2023-11-18T07:05:03.471742Z","shell.execute_reply.started":"2023-11-18T07:05:03.352111Z","shell.execute_reply":"2023-11-18T07:05:03.470148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sid=[s.series_id for s in data.select('series_id').distinct().limit(5).collect()]\nfor sid in sample_sid:\n    print(sid)\n    d=data.filter(data.series_id==sid).toPandas()\n    e=data.filter((data.series_id==sid)&(data.event.between(0,1))).toPandas()\n    fig = plt.figure()\n    fig, ax= plt.subplots(nrows=2,ncols=1,sharex=True,figsize=(30,10))\n    ax[0].plot(d['timestamp'],d['anglez'])\n    ax[0].axhline(y=0, color=\"black\", linestyle=\"--\")\n    ax[1].plot(d['timestamp'],d['enmo'])\n    for ev, ts in zip(e.event.to_list(),e.timestamp.to_list()):\n        ax[0].axvline(x=ts,color='red' if ev==0 else 'green',linewidth=3,linestyle=\"--\" )\n        ax[1].axvline(x=ts,color='red' if ev==0 else 'green',linewidth=3,linestyle=\"--\")\n    ax[1].xaxis.set_tick_params(rotation=90)\n    ax[1].xaxis.set_major_locator(mdates.DayLocator(interval=1))\n    ax[1].xaxis.set_minor_locator(mdates.HourLocator(interval=4))","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:05:03.474370Z","iopub.execute_input":"2023-11-18T07:05:03.474976Z","iopub.status.idle":"2023-11-18T07:14:20.497191Z","shell.execute_reply.started":"2023-11-18T07:05:03.474931Z","shell.execute_reply":"2023-11-18T07:14:20.495865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\nt_id=test.series_id.unique()[-1]\nsample=test.loc[test.series_id==t_id]\nsample['timestamp']=pd.to_datetime(sample.timestamp)\nsample.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:14:20.500700Z","iopub.execute_input":"2023-11-18T07:14:20.501108Z","iopub.status.idle":"2023-11-18T07:14:20.898261Z","shell.execute_reply.started":"2023-11-18T07:14:20.501075Z","shell.execute_reply":"2023-11-18T07:14:20.897024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax =plt.subplots(figsize=(30,5),sharex=True,nrows=2,ncols=1)\nax[1].xaxis.set_tick_params(rotation=90)\nax[0].plot(sample['timestamp'],sample['anglez'])\nax[1].plot(sample['timestamp'],sample['enmo'])","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:14:20.900452Z","iopub.execute_input":"2023-11-18T07:14:20.900895Z","iopub.status.idle":"2023-11-18T07:14:21.623954Z","shell.execute_reply.started":"2023-11-18T07:14:20.900857Z","shell.execute_reply":"2023-11-18T07:14:21.622676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Analyzing enmo and angle 5mins (avg time lies btw 5-20mins) before onset\n'''\nsleep_anglez=[]\nsleep_enmo=[]\nsid=[s.series_id for s in data.select('series_id').distinct().collect()]\nfor id in sid:\n    steps=data.where((col('series_id')==id)&\\\n                     (col('event').between(0,1))).select('step').rdd.map(lambda x: x[0]).collect()\n    for i in range(0,(len(steps)-2),2):\n        onset=steps[i]\n        onset_init=steps[i]-30\n        wakeup=steps[i+1]\n        wakeup_init=steps[i+1]-30\n        onset_df=data.where((col('series_id')==sid[0])&\\\n                         (col('step').between(onset_init,onset)))\n        wakeup_df=data.where((col('series_id')==sid[0])&\\\n                     (col('step').between(wakeup_init,wakeup))) \n                     \n                     \n'''","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:14:21.625825Z","iopub.execute_input":"2023-11-18T07:14:21.627379Z","iopub.status.idle":"2023-11-18T07:14:21.636048Z","shell.execute_reply.started":"2023-11-18T07:14:21.627321Z","shell.execute_reply":"2023-11-18T07:14:21.634911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-11-18T07:17:18.177705Z","iopub.status.idle":"2023-11-18T07:17:18.178156Z","shell.execute_reply.started":"2023-11-18T07:17:18.177944Z","shell.execute_reply":"2023-11-18T07:17:18.177966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}