{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":10080375,"sourceType":"datasetVersion","datasetId":6214381},{"sourceId":185080,"sourceType":"modelInstanceVersion","modelInstanceId":157781,"modelId":180188}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import kagglehub\n\n# # Download latest version\n# path = kagglehub.model_download(\"dhruvanjaria4555/rf_classifier1/scikitLearn/default\")\n\n# print(\"Path to model files:\", path)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:26:56.987244Z","iopub.execute_input":"2024-12-02T17:26:56.988739Z","iopub.status.idle":"2024-12-02T17:26:57.019186Z","shell.execute_reply.started":"2024-12-02T17:26:56.988674Z","shell.execute_reply":"2024-12-02T17:26:57.017628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pip install scikit-learn==1.5.2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:34:08.720418Z","iopub.execute_input":"2024-12-02T17:34:08.721008Z","iopub.status.idle":"2024-12-02T17:34:19.028272Z","shell.execute_reply.started":"2024-12-02T17:34:08.720944Z","shell.execute_reply":"2024-12-02T17:34:19.026608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pip install polars==0.19.2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:34:29.352195Z","iopub.execute_input":"2024-12-02T17:34:29.352644Z","iopub.status.idle":"2024-12-02T17:34:39.525775Z","shell.execute_reply.started":"2024-12-02T17:34:29.352600Z","shell.execute_reply":"2024-12-02T17:34:39.524266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip download polars==0.19.2 -d ./polars/\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:34:39.529092Z","iopub.execute_input":"2024-12-02T17:34:39.529665Z","iopub.status.idle":"2024-12-02T17:34:42.124443Z","shell.execute_reply.started":"2024-12-02T17:34:39.529600Z","shell.execute_reply":"2024-12-02T17:34:42.122969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip download scikit-learn==1.5.2 -d ./scikit-learn/\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:34:42.126210Z","iopub.execute_input":"2024-12-02T17:34:42.126630Z","iopub.status.idle":"2024-12-02T17:34:45.861892Z","shell.execute_reply.started":"2024-12-02T17:34:42.126589Z","shell.execute_reply":"2024-12-02T17:34:45.860211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# from zipfile import ZipFile\n\n# dirName = \"./\"\n# zipName = \"packages.zip\"\n\n# # Create a ZipFile Object\n# with ZipFile(zipName, 'w') as zipObj:\n#     # Iterate over all the files in directory\n#     for folderName, subfolders, filenames in os.walk(dirName):\n#         for filename in filenames:\n#             if (filename != zipName):\n#                 # create complete filepath of file in directory\n#                 filePath = os.path.join(folderName, filename)\n#                 # Add file to zip\n#                 zipObj.write(filePath)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:38:49.000887Z","iopub.execute_input":"2024-12-02T17:38:49.001409Z","iopub.status.idle":"2024-12-02T17:38:49.331121Z","shell.execute_reply.started":"2024-12-02T17:38:49.001352Z","shell.execute_reply":"2024-12-02T17:38:49.330143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"! ls ../input/rf-depend/depandencies_rf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:44:48.863856Z","iopub.execute_input":"2024-12-02T17:44:48.864268Z","iopub.status.idle":"2024-12-02T17:44:50.050804Z","shell.execute_reply.started":"2024-12-02T17:44:48.864228Z","shell.execute_reply":"2024-12-02T17:44:50.049283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install scikit-learn==1.5.2 --no-index --find-links=file:///kaggle/input/rf-depend/depandencies_rf/\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:45:37.050516Z","iopub.execute_input":"2024-12-02T17:45:37.050969Z","iopub.status.idle":"2024-12-02T17:45:47.621494Z","shell.execute_reply.started":"2024-12-02T17:45:37.050926Z","shell.execute_reply":"2024-12-02T17:45:47.620186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install polars==0.19.2 --no-index --find-links=file:///kaggle/input/rf-depend/depandencies_rf/\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:45:53.854631Z","iopub.execute_input":"2024-12-02T17:45:53.855055Z","iopub.status.idle":"2024-12-02T17:46:04.376227Z","shell.execute_reply.started":"2024-12-02T17:45:53.855016Z","shell.execute_reply":"2024-12-02T17:46:04.374891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pickle\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport datetime \nfrom tqdm import tqdm\nimport sklearn\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:26:57.045318Z","iopub.execute_input":"2024-12-02T17:26:57.045842Z","iopub.status.idle":"2024-12-02T17:27:01.149606Z","shell.execute_reply.started":"2024-12-02T17:26:57.045788Z","shell.execute_reply":"2024-12-02T17:27:01.147781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open('/kaggle/input/rf_classifier1/scikitlearn/default/1/rf_classifier_5m_8h.pkl', 'rb') as f:\n    rf_classifier = pickle.load(f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:27:01.151133Z","iopub.execute_input":"2024-12-02T17:27:01.151780Z","iopub.status.idle":"2024-12-02T17:27:02.068236Z","shell.execute_reply.started":"2024-12-02T17:27:01.151739Z","shell.execute_reply":"2024-12-02T17:27:02.066324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt_transforms = [\n    pl.col('timestamp').str.to_datetime(), \n    (pl.col('timestamp').str.to_datetime().dt.year()-2000).cast(pl.UInt8).alias('year'), \n    pl.col('timestamp').str.to_datetime().dt.month().cast(pl.UInt8).alias('month'),\n    pl.col('timestamp').str.to_datetime().dt.day().cast(pl.UInt8).alias('day'), \n    pl.col('timestamp').str.to_datetime().dt.hour().cast(pl.UInt8).alias('hour')\n]\n\ndata_transforms = [\n    pl.col('anglez').cast(pl.Int16), # Casting anglez to 16 bit integer\n    (pl.col('enmo')*1000).cast(pl.UInt16), # Convert enmo to 16 bit uint\n]\n\ntest_series = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet').with_columns(\n    dt_transforms + data_transforms\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:27:02.069643Z","iopub.status.idle":"2024-12-02T17:27:02.070031Z","shell.execute_reply.started":"2024-12-02T17:27:02.069852Z","shell.execute_reply":"2024-12-02T17:27:02.069871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features, feature_cols = [pl.col('hour')], ['hour']\n\nfor mins in [5, 30, 60*2, 60*8] :\n    \n    for var in ['enmo', 'anglez'] :\n        \n        features += [\n            pl.col(var).rolling_mean(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_mean'),\n            pl.col(var).rolling_max(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_max'),\n            pl.col(var).rolling_std(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_std')\n        ]\n\n        feature_cols += [ \n            f'{var}_{mins}m_mean', f'{var}_{mins}m_max', f'{var}_{mins}m_std'\n        ]\n\n        # Getting first variations\n        features += [\n            (pl.col(var).diff().abs().rolling_mean(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_mean'),\n            (pl.col(var).diff().abs().rolling_max(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_max'),\n            (pl.col(var).diff().abs().rolling_std(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_std')\n        ]\n\n        feature_cols += [ \n            f'{var}_1v_{mins}m_mean', f'{var}_1v_{mins}m_max', f'{var}_1v_{mins}m_std'\n        ]\n\nid_cols = ['series_id', 'step', 'timestamp']\n\ntest_series = test_series.with_columns(\n    features\n).select(id_cols + feature_cols)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:27:02.071133Z","iopub.status.idle":"2024-12-02T17:27:02.071737Z","shell.execute_reply.started":"2024-12-02T17:27:02.071414Z","shell.execute_reply":"2024-12-02T17:27:02.071456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_events(series, classifier) :\n    '''\n    Takes a time series and a classifier and returns a formatted submission dataframe.\n    '''\n    \n    series_ids = series['series_id'].unique(maintain_order=True).to_list()\n    events = pl.DataFrame(schema={'series_id':str, 'step':int, 'event':str, 'score':float})\n\n    for idx in tqdm(series_ids) : \n\n        # Collecting sample and normalizing features\n        scale_cols = [col for col in feature_cols if (col != 'hour') & (series[col].std() !=0)]\n        X = series.filter(pl.col('series_id') == idx).select(id_cols + feature_cols).with_columns(\n            [(pl.col(col) / series[col].std()).cast(pl.Float32) for col in scale_cols]\n        )\n\n        # Applying classifier to get predictions and scores\n        preds, probs = classifier.predict(X[feature_cols]), classifier.predict_proba(X[feature_cols])[:, 1]\n\n        #NOTE: Considered using rolling max to get sleep periods excluding <30 min interruptions, but ended up decreasing performance\n        X = X.with_columns(\n            pl.lit(preds).cast(pl.Int8).alias('prediction'), \n            pl.lit(probs).alias('probability')\n                        )\n        \n        # Getting predicted onset and wakeup time steps\n        pred_onsets = X.filter(X['prediction'].diff() > 0)['step'].to_list()\n        pred_wakeups = X.filter(X['prediction'].diff() < 0)['step'].to_list()\n        \n        if len(pred_onsets) > 0 : \n            \n            # Ensuring all predicted sleep periods begin and end\n            if min(pred_wakeups) < min(pred_onsets) : \n                pred_wakeups = pred_wakeups[1:]\n\n            if max(pred_onsets) > max(pred_wakeups) :\n                pred_onsets = pred_onsets[:-1]\n\n            # Keeping sleep periods longer than 30 minutes\n            sleep_periods = [(onset, wakeup) for onset, wakeup in zip(pred_onsets, pred_wakeups) if wakeup - onset >= 12 * 30]\n\n            for onset, wakeup in sleep_periods :\n                # Scoring using mean probability over period\n                score = X.filter((pl.col('step') >= onset) & (pl.col('step') <= wakeup))['probability'].mean()\n\n                # Adding sleep event to dataframe\n                events = events.vstack(pl.DataFrame().with_columns(\n                    pl.Series([idx, idx]).alias('series_id'), \n                    pl.Series([onset, wakeup]).alias('step'),\n                    pl.Series(['onset', 'wakeup']).alias('event'),\n                    pl.Series([score, score]).alias('score')\n                ))\n\n    # Adding row id column\n    events = events.to_pandas().reset_index().rename(columns={'index':'row_id'})\n\n    return events","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:27:02.075511Z","iopub.status.idle":"2024-12-02T17:27:02.076162Z","shell.execute_reply.started":"2024-12-02T17:27:02.075867Z","shell.execute_reply":"2024-12-02T17:27:02.075900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = get_events(test_series.collect(), rf_classifier)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T17:27:02.077263Z","iopub.status.idle":"2024-12-02T17:27:02.077872Z","shell.execute_reply.started":"2024-12-02T17:27:02.077568Z","shell.execute_reply":"2024-12-02T17:27:02.077600Z"}},"outputs":[],"execution_count":null}]}