{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nimport gc\nfrom tqdm.auto import tqdm \n\nimport xgboost as xgb\nimport lightgbm as lgb\nimport pickle\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-07T14:30:35.526438Z","iopub.execute_input":"2023-11-07T14:30:35.527134Z","iopub.status.idle":"2023-11-07T14:30:35.541017Z","shell.execute_reply.started":"2023-11-07T14:30:35.527102Z","shell.execute_reply":"2023-11-07T14:30:35.540215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.read_parquet('/kaggle/input/make-data/train_luan.parquet')\n# gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:17:56.248674Z","iopub.execute_input":"2023-11-07T15:17:56.249049Z","iopub.status.idle":"2023-11-07T15:18:04.791055Z","shell.execute_reply.started":"2023-11-07T15:17:56.249020Z","shell.execute_reply":"2023-11-07T15:18:04.789822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# prepare data","metadata":{}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-11-06T13:51:30.566537Z","iopub.execute_input":"2023-11-06T13:51:30.567008Z","iopub.status.idle":"2023-11-06T13:51:30.618810Z","shell.execute_reply.started":"2023-11-06T13:51:30.566972Z","shell.execute_reply":"2023-11-06T13:51:30.617922Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def luan_unique(seq):\n    seen = set()\n    seen_add = seen.add\n    return [x for x in seq if not (x in seen or seen_add(x))]\n\n\nids = luan_unique(train['series_id'])","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:18:04.793153Z","iopub.execute_input":"2023-11-07T15:18:04.793622Z","iopub.status.idle":"2023-11-07T15:18:07.069324Z","shell.execute_reply.started":"2023-11-07T15:18:04.793579Z","shell.execute_reply":"2023-11-07T15:18:07.068292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validate year month day hour\n\ndict_ids2data = {}\n\nfor i in tqdm(range(len(ids))):\n    print (i,': ', ids[i])\n    for i_c in ['year', 'month', 'day', 'hour']:\n    # for i in range(2):\n    #     dict_ids2data[ids[i]] = train_time_extract.filter(pl.col(\"series_id\") == ids[i])\n        df_id = train.filter(pl.col(\"series_id\") == ids[i])\n        diff = (df_id[i_c][1:] - df_id[i_c][:-1])\n#         if (len(diff.unique())) > 3:\n        print (i_c,' '.join([str(x) for x in diff.unique()]))\n\n\ndel dict_ids2data\ngc.collect()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:18:07.070964Z","iopub.execute_input":"2023-11-07T15:18:07.071262Z","iopub.status.idle":"2023-11-07T15:18:27.606773Z","shell.execute_reply.started":"2023-11-07T15:18:07.071236Z","shell.execute_reply":"2023-11-07T15:18:27.605721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validate year month day hour\n# anglez enmo\n\ndict_ids2data = {}\n\n# for i in range(2):\nfor i in tqdm(range(len(ids))):\n    print (i,': ', ids[i])\n    dict_ids2data[ids[i]] = train.filter(pl.col(\"series_id\") == ids[i])\n    \n    dict_ids2data[ids[i]] = dict_ids2data[ids[i]].with_columns(\n        anglez_lag_1 = dict_ids2data[ids[i]]['anglez'].shift(periods=1),\n        anglez_lag_2 = dict_ids2data[ids[i]]['anglez'].shift(periods=2),\n        anglez_lag_3 = dict_ids2data[ids[i]]['anglez'].shift(periods=3),\n\n        enmo_lag_1 = dict_ids2data[ids[i]]['enmo'].shift(periods=1),\n        enmo_lag_2 = dict_ids2data[ids[i]]['enmo'].shift(periods=2),\n        enmo_lag_3 = dict_ids2data[ids[i]]['enmo'].shift(periods=3)\n    )\n    \ntrain = pl.concat([i for i in dict_ids2data.values()], rechunk=True).drop_nulls()\n\n\ndel dict_ids2data\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:18:38.304397Z","iopub.execute_input":"2023-11-07T15:18:38.304813Z","iopub.status.idle":"2023-11-07T15:18:49.506214Z","shell.execute_reply.started":"2023-11-07T15:18:38.304778Z","shell.execute_reply":"2023-11-07T15:18:49.505182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.write_parquet(\"train_data.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:20:57.701979Z","iopub.execute_input":"2023-11-07T15:20:57.702359Z","iopub.status.idle":"2023-11-07T15:21:11.701769Z","shell.execute_reply.started":"2023-11-07T15:20:57.702329Z","shell.execute_reply":"2023-11-07T15:21:11.700800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_every12 = train.take_every(12)\nlen(train_every12)","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:18:49.507847Z","iopub.execute_input":"2023-11-07T15:18:49.508161Z","iopub.status.idle":"2023-11-07T15:18:49.734108Z","shell.execute_reply.started":"2023-11-07T15:18:49.508134Z","shell.execute_reply":"2023-11-07T15:18:49.733295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dict_ids2data = {}\n\ns = 0\nfor i in tqdm(range(len(ids))):\n    tmp_len = len(train_every12.filter(pl.col(\"series_id\") == ids[i]))\n    s += tmp_len\n    print (i,':\\t', ids[i],'\\t',tmp_len,'\\t', s)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-07T14:11:56.292443Z","iopub.execute_input":"2023-11-07T14:11:56.292836Z","iopub.status.idle":"2023-11-07T14:11:56.783838Z","shell.execute_reply.started":"2023-11-07T14:11:56.292807Z","shell.execute_reply":"2023-11-07T14:11:56.782362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# series_id\tstep\ttimestamp\n\ndef split_train_test(df, ids, n_train):\n    dict_ids2data = {}\n\n    # for i in range(2):\n    for i in tqdm(range(len(ids))):\n        print (i,': ', ids[i])\n        dict_ids2data[ids[i]] = df.filter(pl.col(\"series_id\") == ids[i])\n\n    train_ = pl.concat([i for i, i_id in zip(dict_ids2data.values(), range(len(ids))) if i_id <= n_train], rechunk=True)\n    test_ = pl.concat([i for i, i_id in zip(dict_ids2data.values(), range(len(ids))) if i_id > n_train], rechunk=True)\n\n    del dict_ids2data\n    gc.collect()\n    \n    return train_, test_\n\n\n\ntrain_data_12, test_data_12 = split_train_test(train_every12, ids, 30)","metadata":{"execution":{"iopub.status.busy":"2023-11-07T14:12:14.022648Z","iopub.execute_input":"2023-11-07T14:12:14.023351Z","iopub.status.idle":"2023-11-07T14:12:14.806875Z","shell.execute_reply.started":"2023-11-07T14:12:14.023293Z","shell.execute_reply":"2023-11-07T14:12:14.805908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_train_dataset(df):\n    if ('step' in df.columns):\n        df = df.drop(\"step\")\n    if ('series_id' in df.columns):\n        df = df.drop(\"series_id\")\n    if ('timestamp' in df.columns):\n        df = df.drop(\"timestamp\")\n    return df.drop(\"awake\"), df['awake']\n\nX_train, y_train = make_train_dataset(train_data_12)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-07T14:12:17.695088Z","iopub.execute_input":"2023-11-07T14:12:17.695552Z","iopub.status.idle":"2023-11-07T14:12:17.708488Z","shell.execute_reply.started":"2023-11-07T14:12:17.695513Z","shell.execute_reply":"2023-11-07T14:12:17.707201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_data_12    \ndel train_every12\n# del train_data\n\ngc.collect()\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-07T14:12:21.672541Z","iopub.execute_input":"2023-11-07T14:12:21.672869Z","iopub.status.idle":"2023-11-07T14:12:21.752532Z","shell.execute_reply.started":"2023-11-07T14:12:21.672843Z","shell.execute_reply":"2023-11-07T14:12:21.751351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train lightgbm","metadata":{}},{"cell_type":"code","source":"X_train.columns","metadata":{"execution":{"iopub.status.busy":"2023-11-07T14:12:38.990744Z","iopub.execute_input":"2023-11-07T14:12:38.991122Z","iopub.status.idle":"2023-11-07T14:12:38.998803Z","shell.execute_reply.started":"2023-11-07T14:12:38.991089Z","shell.execute_reply":"2023-11-07T14:12:38.997723Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_events(series, classifier) :\n    '''\n    Takes a time series and a classifier and returns a formatted submission dataframe.\n    '''\n    c_name = series.columns\n\n    id_cols = ['series_id', 'step', 'timestamp']\n    features, feature_cols = [pl.col(i) for i in c_name if i not in id_cols], [i for i in c_name if i not in id_cols]\n    \n    print('len: ', len(feature_cols))\n    \n    series_ids = series['series_id'].unique(maintain_order=True).to_list()\n    events = pl.DataFrame(schema={'series_id':str, 'step':int, 'event':str, 'score':float})\n\n    for idx in tqdm(series_ids) : \n\n        # Collecting sample and normalizing features\n#         scale_cols = [col for col in feature_cols if (col != 'hour') & (series[col].std() !=0)]\n        X = series.filter(pl.col('series_id') == idx)\n#         .select(id_cols + feature_cols).with_columns(\n#             [(pl.col(col) / series[col].std()).cast(pl.Float32) for col in scale_cols]\n#         )\n\n        # Applying classifier to get predictions and scores\n#         print('X\\n', X)\n\n            \n        \n        preds, probs = classifier.predict(X[feature_cols]), classifier.predict_proba(X[feature_cols])[:, 1]\n\n        print('probs\\n', probs)\n        print('preds\\n', preds)\n        \n        \n        print('preds\\n', np.unique(preds))\n\n       \n        \n        #NOTE: Considered using rolling max to get sleep periods excluding <30 min interruptions, but ended up decreasing performance\n        X = X.with_columns(\n            pl.lit(preds).cast(pl.Int8).alias('prediction'), \n            pl.lit(probs).alias('probability')\n                        )\n        \n#         print('X\\n', X)\n\n        \n        # Getting predicted onset and wakeup time steps\n        pred_onsets = X.filter(X['prediction'].diff() > 0)['step'].to_list()\n        pred_wakeups = X.filter(X['prediction'].diff() < 0)['step'].to_list()\n\n        print('pred_onsets\\n', pred_onsets)\n        \n        print('pred_wakeups\\n', pred_wakeups)\n        \n        if len(pred_onsets) > 0 : \n            \n            # Ensuring all predicted sleep periods begin and end\n            if min(pred_wakeups) < min(pred_onsets) : \n                pred_wakeups = pred_wakeups[1:]\n\n            if max(pred_onsets) > max(pred_wakeups) :\n                pred_onsets = pred_onsets[:-1]\n\n            # Keeping sleep periods longer than 30 minutes\n            sleep_periods = [(onset, wakeup) for onset, wakeup in zip(pred_onsets, pred_wakeups) if wakeup - onset >= 12 * 30]\n\n            for onset, wakeup in sleep_periods :\n                # Scoring using mean probability over period\n                score = X.filter((pl.col('step') >= onset) & (pl.col('step') <= wakeup))['probability'].mean()\n\n                # Adding sleep event to dataframe\n                events = events.vstack(pl.DataFrame().with_columns(\n                    pl.Series([idx, idx]).alias('series_id'), \n                    pl.Series([onset, wakeup]).alias('step'),\n                    pl.Series(['onset', 'wakeup']).alias('event'),\n                    pl.Series([score, score]).alias('score')\n                ))\n\n    # Adding row id column\n    events = events.to_pandas().reset_index().rename(columns={'index':'row_id'})\n\n    return events","metadata":{"execution":{"iopub.status.busy":"2023-11-07T14:12:47.209573Z","iopub.execute_input":"2023-11-07T14:12:47.209962Z","iopub.status.idle":"2023-11-07T14:12:47.224436Z","shell.execute_reply.started":"2023-11-07T14:12:47.209931Z","shell.execute_reply":"2023-11-07T14:12:47.223299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_opt =  {\n    'num_leaves': 204,\n    'learning_rate': 0.07649523437092402,\n    'random_state': 42\n}\n\nmy_classifier = lgb.LGBMClassifier(**lgb_opt)\nmy_classifier.fit(X_train, y_train)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-06T15:17:19.436310Z","iopub.execute_input":"2023-11-06T15:17:19.436729Z","iopub.status.idle":"2023-11-06T15:18:00.833792Z","shell.execute_reply.started":"2023-11-06T15:17:19.436699Z","shell.execute_reply":"2023-11-06T15:18:00.832772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# my_classifier = pickle.load(open('/kaggle/input/cmi-lightgbm/trained_model.pkl', 'rb'))\n\nfile = 'trained_model.pkl'\npickle.dump(my_classifier, open(file, 'wb'))","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:21:44.496164Z","iopub.execute_input":"2023-11-07T15:21:44.496549Z","iopub.status.idle":"2023-11-07T15:21:44.574755Z","shell.execute_reply.started":"2023-11-07T15:21:44.496520Z","shell.execute_reply":"2023-11-07T15:21:44.573581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_data_12.drop(\"awake\")\ny_test = test_data_12[\"awake\"]\n\n# Getting event predictions for test set and saving submission\nsubmission = get_events(X_test, my_classifier)\nsubmission.to_csv('submission.csv', index=False)\n\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-11-06T16:38:41.540344Z","iopub.execute_input":"2023-11-06T16:38:41.540777Z","iopub.status.idle":"2023-11-06T16:38:42.326063Z","shell.execute_reply.started":"2023-11-06T16:38:41.540744Z","shell.execute_reply":"2023-11-06T16:38:42.324614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_test = test_data_12.drop(\"awake\")\n# y_test = test_data_12[\"awake\"]\n\nX_test, y_test = make_train_dataset(test_data_12)\n\ny_predict_prob = my_classifier.predict_proba(X_test)\ny_predict = my_classifier.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-11-07T14:27:18.411621Z","iopub.execute_input":"2023-11-07T14:27:18.411995Z","iopub.status.idle":"2023-11-07T14:27:18.987443Z","shell.execute_reply.started":"2023-11-07T14:27:18.411967Z","shell.execute_reply":"2023-11-07T14:27:18.986383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the predictions\n#\ny_pred = y_predict\n#\n# Calculate the confusion matrix\n#\nconf_matrix = confusion_matrix(y_true=y_test, y_pred=y_pred)\n#\n# Print the confusion matrix using Matplotlib\n#\nfig, ax = plt.subplots(figsize=(5, 5))\nax.matshow(conf_matrix, cmap=plt.cm.Blues, alpha=0.3)\nfor i in range(conf_matrix.shape[0]):\n    for j in range(conf_matrix.shape[1]):\n        ax.text(x=j, y=i,s=conf_matrix[i, j], va='center', ha='center', size='xx-large')\n \nplt.xlabel('Predictions')\nplt.ylabel('Actuals')\n# plt.title('Confusion Matrix', fontsize=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:01:45.204848Z","iopub.execute_input":"2023-11-07T15:01:45.205241Z","iopub.status.idle":"2023-11-07T15:01:45.485686Z","shell.execute_reply.started":"2023-11-07T15:01:45.205211Z","shell.execute_reply":"2023-11-07T15:01:45.484372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,8))\nplt.plot(y_predict_prob[10000:17000,1], color = 'r')\nplt.plot(y_test[10000:17000], color = 'b', linestyle = 'dashed')\n\nplt.show()\n\nplt.figure(figsize=(20,8))\nplt.plot(y_predict_prob[:70000,1], color = 'r')\nplt.plot(y_test[:70000], color = 'b', linestyle = 'dashed')\nplt.show()\n\n\nplt.figure(figsize=(20,8))\nplt.plot(y_predict_prob[:,1], color = 'r')\nplt.plot(y_test[:], color = 'b', linestyle = 'dashed')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:11:38.800340Z","iopub.execute_input":"2023-11-07T15:11:38.800757Z","iopub.status.idle":"2023-11-07T15:11:39.897397Z","shell.execute_reply.started":"2023-11-07T15:11:38.800726Z","shell.execute_reply":"2023-11-07T15:11:39.896352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create a DataFrame to store feature names and their importances\nfeature_importance_df = pd.DataFrame({'Feature': X_train.columns, 'Importance': my_classifier.feature_importances_})\n\n# Sort the DataFrame by importance in descending order\nfeature_importance_df = feature_importance_df.sort_values(by='Importance', ascending=False)\n\n# Create a bar plot to visualize feature importances\nplt.figure(figsize=(12, 14))\nplt.barh(feature_importance_df['Feature'], feature_importance_df['Importance'], color='skyblue')\nplt.xlabel('Feature Importance')\nplt.ylabel('Feature')\nplt.title('LightXGBoost Feature Importances')\nplt.gca().invert_yaxis()  # Invert the y-axis to display the most important features at the top\nplt.show()\ndisplay(feature_importance_df)","metadata":{"execution":{"iopub.status.busy":"2023-11-07T15:12:54.837766Z","iopub.execute_input":"2023-11-07T15:12:54.838170Z","iopub.status.idle":"2023-11-07T15:12:55.665435Z","shell.execute_reply.started":"2023-11-07T15:12:54.838140Z","shell.execute_reply":"2023-11-07T15:12:55.664401Z"},"trusted":true},"execution_count":null,"outputs":[]}]}