{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30788,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:50.881891Z","iopub.execute_input":"2024-11-21T17:54:50.882308Z","iopub.status.idle":"2024-11-21T17:54:50.886918Z","shell.execute_reply.started":"2024-11-21T17:54:50.882274Z","shell.execute_reply":"2024-11-21T17:54:50.886144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/brist1d/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/brist1d/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:50.888294Z","iopub.execute_input":"2024-11-21T17:54:50.888551Z","iopub.status.idle":"2024-11-21T17:54:58.762452Z","shell.execute_reply.started":"2024-11-21T17:54:50.888526Z","shell.execute_reply":"2024-11-21T17:54:58.761663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Shape of train ',train.shape)\nprint(f'shape of test {test.shape} ')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:58.763932Z","iopub.execute_input":"2024-11-21T17:54:58.764207Z","iopub.status.idle":"2024-11-21T17:54:58.768968Z","shell.execute_reply.started":"2024-11-21T17:54:58.764181Z","shell.execute_reply":"2024-11-21T17:54:58.768288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.sample(4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:58.769807Z","iopub.execute_input":"2024-11-21T17:54:58.770036Z","iopub.status.idle":"2024-11-21T17:54:58.802294Z","shell.execute_reply.started":"2024-11-21T17:54:58.770013Z","shell.execute_reply":"2024-11-21T17:54:58.801692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.sample(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:58.820794Z","iopub.execute_input":"2024-11-21T17:54:58.821036Z","iopub.status.idle":"2024-11-21T17:54:58.838032Z","shell.execute_reply.started":"2024-11-21T17:54:58.821012Z","shell.execute_reply":"2024-11-21T17:54:58.837373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info(show_counts=True,verbose = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:58.838838Z","iopub.execute_input":"2024-11-21T17:54:58.839081Z","iopub.status.idle":"2024-11-21T17:54:59.324654Z","shell.execute_reply.started":"2024-11-21T17:54:58.839057Z","shell.execute_reply":"2024-11-21T17:54:59.323969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols=[]\nfor i in train.columns:\n    col = i.split('-')[0]\n    uniq_cols.append(col)\nuniq_cols = np.unique(uniq_cols)\nprint(uniq_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:59.325585Z","iopub.execute_input":"2024-11-21T17:54:59.325914Z","iopub.status.idle":"2024-11-21T17:54:59.331374Z","shell.execute_reply.started":"2024-11-21T17:54:59.325883Z","shell.execute_reply":"2024-11-21T17:54:59.330678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols=[]\nfor i in train.columns:\n    col = i.split('-')[0]\n    uniq_cols.append(col)\nuniq_cols = np.unique(uniq_cols)\nprint(uniq_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:59.332343Z","iopub.execute_input":"2024-11-21T17:54:59.332607Z","iopub.status.idle":"2024-11-21T17:54:59.343593Z","shell.execute_reply.started":"2024-11-21T17:54:59.332581Z","shell.execute_reply":"2024-11-21T17:54:59.343008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols=[]\nfor i in test.columns:\n    col = i.split('-')[0]\n    uniq_cols.append(col)\nuniq_cols = np.unique(uniq_cols)\nprint(uniq_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:59.344418Z","iopub.execute_input":"2024-11-21T17:54:59.344638Z","iopub.status.idle":"2024-11-21T17:54:59.356791Z","shell.execute_reply.started":"2024-11-21T17:54:59.344616Z","shell.execute_reply":"2024-11-21T17:54:59.356111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tabulate import tabulate\nnulls_train = train.isnull().sum()\nnull_percent_train = train.isnull().mean() * 100\n# Prepare data for the table\ntable_data = list(zip(nulls_train.index, nulls_train.values, null_percent_train.values))\nheaders = ['Column', 'Null Count', 'Null Percentage']\ntable = tabulate(table_data, headers, tablefmt='pretty')\nprint(table)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:54:59.357651Z","iopub.execute_input":"2024-11-21T17:54:59.357904Z","iopub.status.idle":"2024-11-21T17:55:00.272077Z","shell.execute_reply.started":"2024-11-21T17:54:59.357881Z","shell.execute_reply":"2024-11-21T17:55:00.271406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tabulate import tabulate\nnulls_test = test.isnull().sum()\nnull_percent_test = test.isnull().mean() * 100\n# Prepare data for the table\ntable_data = list(zip(nulls_test.index, nulls_test.values, null_percent_test.values))\nheaders = ['Column', 'Null Count', 'Null Percentage']\ntable = tabulate(table_data, headers, tablefmt='pretty')\nprint(table)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.274613Z","iopub.execute_input":"2024-11-21T17:55:00.274889Z","iopub.status.idle":"2024-11-21T17:55:00.322190Z","shell.execute_reply.started":"2024-11-21T17:55:00.274862Z","shell.execute_reply":"2024-11-21T17:55:00.321553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_percent_train.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.323003Z","iopub.execute_input":"2024-11-21T17:55:00.323231Z","iopub.status.idle":"2024-11-21T17:55:00.328270Z","shell.execute_reply.started":"2024-11-21T17:55:00.323207Z","shell.execute_reply":"2024-11-21T17:55:00.327586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_percent_test.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.329055Z","iopub.execute_input":"2024-11-21T17:55:00.329325Z","iopub.status.idle":"2024-11-21T17:55:00.339303Z","shell.execute_reply.started":"2024-11-21T17:55:00.329283Z","shell.execute_reply":"2024-11-21T17:55:00.338628Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## On average 44% values are mising from data","metadata":{}},{"cell_type":"markdown","source":"### On each unique col I am making a separate df to check each features relation with target then startegy for missing values will be developed","metadata":{}},{"cell_type":"code","source":"uniq_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.340131Z","iopub.execute_input":"2024-11-21T17:55:00.340392Z","iopub.status.idle":"2024-11-21T17:55:00.348978Z","shell.execute_reply.started":"2024-11-21T17:55:00.340366Z","shell.execute_reply":"2024-11-21T17:55:00.348370Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.349764Z","iopub.execute_input":"2024-11-21T17:55:00.349993Z","iopub.status.idle":"2024-11-21T17:55:00.358306Z","shell.execute_reply.started":"2024-11-21T17:55:00.349969Z","shell.execute_reply":"2024-11-21T17:55:00.357657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## this function will make a separate df for each col see the result to understand\ndef df_of_col (train,col):\n    df = pd.DataFrame()\n    for i in train.columns:\n        if (i.split('-')[0] ==col) & (i  not in ['hr','bg+1:00','id','p_num']):\n            df[i] = train[i]\n    return df\ndef df_of_col (test,col):\n    df = pd.DataFrame()\n    for i in train.columns:\n        if (i.split('-')[0] ==col) & (i  not in ['hr','bg+1:00','id','p_num']):\n            df[i] = test[i]\n    return df\n       ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.359155Z","iopub.execute_input":"2024-11-21T17:55:00.359404Z","iopub.status.idle":"2024-11-21T17:55:00.366728Z","shell.execute_reply.started":"2024-11-21T17:55:00.359380Z","shell.execute_reply":"2024-11-21T17:55:00.366042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"activity_df = df_of_col(train,'activity')\nactivity_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.367625Z","iopub.execute_input":"2024-11-21T17:55:00.367891Z","iopub.status.idle":"2024-11-21T17:55:00.472337Z","shell.execute_reply.started":"2024-11-21T17:55:00.367857Z","shell.execute_reply":"2024-11-21T17:55:00.471653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.473332Z","iopub.execute_input":"2024-11-21T17:55:00.473615Z","iopub.status.idle":"2024-11-21T17:55:00.478496Z","shell.execute_reply.started":"2024-11-21T17:55:00.473587Z","shell.execute_reply":"2024-11-21T17:55:00.477824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Train\nactivity = df_of_col(train,uniq_cols[0])\nbg =  df_of_col(train,uniq_cols[1])\ncals =  df_of_col(train,uniq_cols[3])\ncarbs =  df_of_col(train,uniq_cols[4])\ninsulin =  df_of_col(train,'insulin')\nsteps =  df_of_col(train,uniq_cols[-2])\ntime =  df_of_col(train,uniq_cols[-1])\n#Test\nactivity = df_of_col(test,uniq_cols[0])\nbg_test =  df_of_col(test,uniq_cols[1])\ncals_test =  df_of_col(test,uniq_cols[3])\ncarbs_test =  df_of_col(test,uniq_cols[4])\ninsulin_test =  df_of_col(test,'insulin')\nsteps_test =  df_of_col(test,uniq_cols[-2])\ntime_test =  df_of_col(test,uniq_cols[-1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:00.479444Z","iopub.execute_input":"2024-11-21T17:55:00.479717Z","iopub.status.idle":"2024-11-21T17:55:01.253950Z","shell.execute_reply.started":"2024-11-21T17:55:00.479689Z","shell.execute_reply":"2024-11-21T17:55:01.253229Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" ## target feature's distribution","metadata":{}},{"cell_type":"code","source":"target = train['bg+1:00']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:01.254902Z","iopub.execute_input":"2024-11-21T17:55:01.255155Z","iopub.status.idle":"2024-11-21T17:55:01.260730Z","shell.execute_reply.started":"2024-11-21T17:55:01.255130Z","shell.execute_reply":"2024-11-21T17:55:01.260123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.distplot(target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:01.261530Z","iopub.execute_input":"2024-11-21T17:55:01.261753Z","iopub.status.idle":"2024-11-21T17:55:02.224882Z","shell.execute_reply.started":"2024-11-21T17:55:01.261730Z","shell.execute_reply":"2024-11-21T17:55:02.224139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'skewness in bg+1:00 is {target.skew()}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:02.225878Z","iopub.execute_input":"2024-11-21T17:55:02.226194Z","iopub.status.idle":"2024-11-21T17:55:02.232431Z","shell.execute_reply.started":"2024-11-21T17:55:02.226163Z","shell.execute_reply":"2024-11-21T17:55:02.231687Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## activity features and target","metadata":{}},{"cell_type":"code","source":"activity_missing_percentage = activity.isnull().mean() * 100\nprint(activity_missing_percentage)\nprint(\"------------------------------------------------------------------------------\")\nprint(f'mean percentage for missing values is {np.mean(activity_missing_percentage)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:02.233362Z","iopub.execute_input":"2024-11-21T17:55:02.233635Z","iopub.status.idle":"2024-11-21T17:55:02.253464Z","shell.execute_reply.started":"2024-11-21T17:55:02.233607Z","shell.execute_reply":"2024-11-21T17:55:02.252660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'on average this much values are missing for activity columns : {np.mean(activity.isnull().sum())}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:02.254458Z","iopub.execute_input":"2024-11-21T17:55:02.254725Z","iopub.status.idle":"2024-11-21T17:55:02.269360Z","shell.execute_reply.started":"2024-11-21T17:55:02.254698Z","shell.execute_reply":"2024-11-21T17:55:02.268700Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### so its better to drop alll columns of activity from training","metadata":{}},{"cell_type":"code","source":"train_preprocessed = train.drop(columns=activity.columns)\ntest_preprocessed = test.drop(columns=activity.columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:02.270215Z","iopub.execute_input":"2024-11-21T17:55:02.270483Z","iopub.status.idle":"2024-11-21T17:55:02.475974Z","shell.execute_reply.started":"2024-11-21T17:55:02.270457Z","shell.execute_reply":"2024-11-21T17:55:02.475086Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## bg and target","metadata":{}},{"cell_type":"code","source":"bg.sample(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:02.477127Z","iopub.execute_input":"2024-11-21T17:55:02.477458Z","iopub.status.idle":"2024-11-21T17:55:02.516705Z","shell.execute_reply.started":"2024-11-21T17:55:02.477426Z","shell.execute_reply":"2024-11-21T17:55:02.515857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average skewness in bg cols are {np.mean(bg.skew())}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:02.517739Z","iopub.execute_input":"2024-11-21T17:55:02.518035Z","iopub.status.idle":"2024-11-21T17:55:02.630639Z","shell.execute_reply.started":"2024-11-21T17:55:02.518005Z","shell.execute_reply":"2024-11-21T17:55:02.629815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(bg['bg-0:20'],target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:02.631688Z","iopub.execute_input":"2024-11-21T17:55:02.631966Z","iopub.status.idle":"2024-11-21T17:55:03.328917Z","shell.execute_reply.started":"2024-11-21T17:55:02.631937Z","shell.execute_reply":"2024-11-21T17:55:03.328250Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship between `bg` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewness in `bg` features is 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot between `bg['bg+0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between bg columns and the target\nbg_corr_spearman = bg.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(bg_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:03.334452Z","iopub.execute_input":"2024-11-21T17:55:03.334711Z","iopub.status.idle":"2024-11-21T17:55:04.352965Z","shell.execute_reply.started":"2024-11-21T17:55:03.334686Z","shell.execute_reply":"2024-11-21T17:55:04.352279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(bg_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:04.353922Z","iopub.execute_input":"2024-11-21T17:55:04.354179Z","iopub.status.idle":"2024-11-21T17:55:04.358490Z","shell.execute_reply.started":"2024-11-21T17:55:04.354154Z","shell.execute_reply":"2024-11-21T17:55:04.357744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nbg_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:04.359344Z","iopub.execute_input":"2024-11-21T17:55:04.359575Z","iopub.status.idle":"2024-11-21T17:55:04.852496Z","shell.execute_reply.started":"2024-11-21T17:55:04.359552Z","shell.execute_reply":"2024-11-21T17:55:04.851817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(bg_corr_spearman)\nprint(bg_corr_spearman[bg_corr_spearman < mean_corr])\nprint(f'bg features with corr less than avg: {len(bg_corr_spearman[bg_corr_spearman < mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:04.853348Z","iopub.execute_input":"2024-11-21T17:55:04.853577Z","iopub.status.idle":"2024-11-21T17:55:04.859871Z","shell.execute_reply.started":"2024-11-21T17:55:04.853553Z","shell.execute_reply":"2024-11-21T17:55:04.859203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = bg_corr_spearman[bg_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)\ntest_preprocessed = test_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:04.860733Z","iopub.execute_input":"2024-11-21T17:55:04.860983Z","iopub.status.idle":"2024-11-21T17:55:05.041961Z","shell.execute_reply.started":"2024-11-21T17:55:04.860958Z","shell.execute_reply":"2024-11-21T17:55:05.041203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bg= bg.drop(columns=features_to_drop.index)\nbg_test= bg_test.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:05.042950Z","iopub.execute_input":"2024-11-21T17:55:05.043205Z","iopub.status.idle":"2024-11-21T17:55:05.069647Z","shell.execute_reply.started":"2024-11-21T17:55:05.043180Z","shell.execute_reply":"2024-11-21T17:55:05.069017Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### dropped features with corr less than average corr","metadata":{}},{"cell_type":"code","source":"pip install missingno","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:05.070503Z","iopub.execute_input":"2024-11-21T17:55:05.070734Z","iopub.status.idle":"2024-11-21T17:55:08.821701Z","shell.execute_reply.started":"2024-11-21T17:55:05.070711Z","shell.execute_reply":"2024-11-21T17:55:08.820695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(bg)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:08.823228Z","iopub.execute_input":"2024-11-21T17:55:08.823568Z","iopub.status.idle":"2024-11-21T17:55:10.320290Z","shell.execute_reply.started":"2024-11-21T17:55:08.823535Z","shell.execute_reply":"2024-11-21T17:55:10.319587Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## cals and target","metadata":{}},{"cell_type":"code","source":"cals.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:10.321232Z","iopub.execute_input":"2024-11-21T17:55:10.321502Z","iopub.status.idle":"2024-11-21T17:55:10.346967Z","shell.execute_reply.started":"2024-11-21T17:55:10.321477Z","shell.execute_reply":"2024-11-21T17:55:10.346281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average skewness in calories cols are {np.mean(cals.skew())}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:10.347937Z","iopub.execute_input":"2024-11-21T17:55:10.348223Z","iopub.status.idle":"2024-11-21T17:55:10.463087Z","shell.execute_reply.started":"2024-11-21T17:55:10.348193Z","shell.execute_reply":"2024-11-21T17:55:10.462378Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship betweencalsbg` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewnesscalsn `bg` feature2.73 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot bcalswecals-`bg['bg+0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between cals columns and the target\ncals_corr_spearman = cals.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(cals_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:10.463976Z","iopub.execute_input":"2024-11-21T17:55:10.464219Z","iopub.status.idle":"2024-11-21T17:55:10.636287Z","shell.execute_reply.started":"2024-11-21T17:55:10.464194Z","shell.execute_reply":"2024-11-21T17:55:10.635612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(cals_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:10.637210Z","iopub.execute_input":"2024-11-21T17:55:10.637525Z","iopub.status.idle":"2024-11-21T17:55:10.641904Z","shell.execute_reply.started":"2024-11-21T17:55:10.637494Z","shell.execute_reply":"2024-11-21T17:55:10.641287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ncals_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:10.642718Z","iopub.execute_input":"2024-11-21T17:55:10.642950Z","iopub.status.idle":"2024-11-21T17:55:11.173666Z","shell.execute_reply.started":"2024-11-21T17:55:10.642926Z","shell.execute_reply":"2024-11-21T17:55:11.172978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(cals_corr_spearman)\nprint(cals_corr_spearman[cals_corr_spearman < mean_corr])\nprint(f'cals features with corr less than avg: {len(cals_corr_spearman[cals_corr_spearman < mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:11.174719Z","iopub.execute_input":"2024-11-21T17:55:11.175031Z","iopub.status.idle":"2024-11-21T17:55:11.181282Z","shell.execute_reply.started":"2024-11-21T17:55:11.175000Z","shell.execute_reply":"2024-11-21T17:55:11.180653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = cals_corr_spearman[cals_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)\ntest_preprocessed = test_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:11.182055Z","iopub.execute_input":"2024-11-21T17:55:11.182288Z","iopub.status.idle":"2024-11-21T17:55:11.349902Z","shell.execute_reply.started":"2024-11-21T17:55:11.182265Z","shell.execute_reply":"2024-11-21T17:55:11.349141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cals= cals.drop(columns=features_to_drop.index)\ncals_test= cals_test.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:11.351128Z","iopub.execute_input":"2024-11-21T17:55:11.351512Z","iopub.status.idle":"2024-11-21T17:55:11.381821Z","shell.execute_reply.started":"2024-11-21T17:55:11.351485Z","shell.execute_reply":"2024-11-21T17:55:11.381180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cals","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:11.382694Z","iopub.execute_input":"2024-11-21T17:55:11.382944Z","iopub.status.idle":"2024-11-21T17:55:11.485274Z","shell.execute_reply.started":"2024-11-21T17:55:11.382918Z","shell.execute_reply":"2024-11-21T17:55:11.484584Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### dropped features with corr less than average corr","metadata":{}},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(cals)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:11.486248Z","iopub.execute_input":"2024-11-21T17:55:11.486517Z","iopub.status.idle":"2024-11-21T17:55:12.993256Z","shell.execute_reply.started":"2024-11-21T17:55:11.486491Z","shell.execute_reply":"2024-11-21T17:55:12.992563Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# carbs and target","metadata":{}},{"cell_type":"code","source":"print(f'average skewness in carbs cols are {np.mean(carbs.skew())}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:12.994281Z","iopub.execute_input":"2024-11-21T17:55:12.994534Z","iopub.status.idle":"2024-11-21T17:55:13.109813Z","shell.execute_reply.started":"2024-11-21T17:55:12.994509Z","shell.execute_reply":"2024-11-21T17:55:13.109085Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship between `carbs` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewness in `carbs` feature6.83 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot between `carb-['carbs+0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between carbs columns and the target\ncarbs_corr_spearman = carbs.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(carbs_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:13.110825Z","iopub.execute_input":"2024-11-21T17:55:13.111109Z","iopub.status.idle":"2024-11-21T17:55:14.025037Z","shell.execute_reply.started":"2024-11-21T17:55:13.111082Z","shell.execute_reply":"2024-11-21T17:55:14.024248Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(carbs_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:14.026144Z","iopub.execute_input":"2024-11-21T17:55:14.026467Z","iopub.status.idle":"2024-11-21T17:55:14.030864Z","shell.execute_reply.started":"2024-11-21T17:55:14.026436Z","shell.execute_reply":"2024-11-21T17:55:14.030270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ncarbs_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:14.031917Z","iopub.execute_input":"2024-11-21T17:55:14.032178Z","iopub.status.idle":"2024-11-21T17:55:14.555891Z","shell.execute_reply.started":"2024-11-21T17:55:14.032152Z","shell.execute_reply":"2024-11-21T17:55:14.555203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(carbs_corr_spearman)\nprint(carbs_corr_spearman[carbs_corr_spearman < mean_corr])\nprint(f'carbs features with corr less than avg: {len(carbs_corr_spearman[carbs_corr_spearman < mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:14.556792Z","iopub.execute_input":"2024-11-21T17:55:14.557055Z","iopub.status.idle":"2024-11-21T17:55:14.563515Z","shell.execute_reply.started":"2024-11-21T17:55:14.557029Z","shell.execute_reply":"2024-11-21T17:55:14.562868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = carbs_corr_spearman[carbs_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)\ntest_preprocessed = test_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:14.564377Z","iopub.execute_input":"2024-11-21T17:55:14.564607Z","iopub.status.idle":"2024-11-21T17:55:14.717046Z","shell.execute_reply.started":"2024-11-21T17:55:14.564583Z","shell.execute_reply":"2024-11-21T17:55:14.716182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"carbs= carbs.drop(columns=features_to_drop.index)\ncarbs_test= carbs_test.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:14.718209Z","iopub.execute_input":"2024-11-21T17:55:14.718540Z","iopub.status.idle":"2024-11-21T17:55:14.749052Z","shell.execute_reply.started":"2024-11-21T17:55:14.718509Z","shell.execute_reply":"2024-11-21T17:55:14.747967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(carbs)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:14.750107Z","iopub.execute_input":"2024-11-21T17:55:14.750359Z","iopub.status.idle":"2024-11-21T17:55:17.066631Z","shell.execute_reply.started":"2024-11-21T17:55:14.750335Z","shell.execute_reply":"2024-11-21T17:55:17.065938Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# insulin and target","metadata":{}},{"cell_type":"code","source":"print(f'average skewness in insulin cols are {np.mean(insulin.skew())}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:17.067612Z","iopub.execute_input":"2024-11-21T17:55:17.067913Z","iopub.status.idle":"2024-11-21T17:55:17.174269Z","shell.execute_reply.started":"2024-11-21T17:55:17.067884Z","shell.execute_reply":"2024-11-21T17:55:17.173497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(insulin['insulin-0:20'],target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:17.175223Z","iopub.execute_input":"2024-11-21T17:55:17.175509Z","iopub.status.idle":"2024-11-21T17:55:17.894452Z","shell.execute_reply.started":"2024-11-21T17:55:17.175482Z","shell.execute_reply":"2024-11-21T17:55:17.893781Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship between `insulin` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewness in `insulin` features4 52 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot between `insulin['insulin-0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between insulin columns and the target\ninsulin_corr_spearman = insulin.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(insulin_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:17.895364Z","iopub.execute_input":"2024-11-21T17:55:17.895610Z","iopub.status.idle":"2024-11-21T17:55:19.524417Z","shell.execute_reply.started":"2024-11-21T17:55:17.895586Z","shell.execute_reply":"2024-11-21T17:55:19.523472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(insulin_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:19.525567Z","iopub.execute_input":"2024-11-21T17:55:19.525859Z","iopub.status.idle":"2024-11-21T17:55:19.530221Z","shell.execute_reply.started":"2024-11-21T17:55:19.525824Z","shell.execute_reply":"2024-11-21T17:55:19.529491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ninsulin_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:19.531106Z","iopub.execute_input":"2024-11-21T17:55:19.531365Z","iopub.status.idle":"2024-11-21T17:55:20.101069Z","shell.execute_reply.started":"2024-11-21T17:55:19.531339Z","shell.execute_reply":"2024-11-21T17:55:20.100373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(insulin_corr_spearman)\nprint(insulin_corr_spearman[insulin_corr_spearman < mean_corr])\nprint(f'insulin features with corr less than avg: {len(insulin_corr_spearman[insulin_corr_spearman< mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:20.101979Z","iopub.execute_input":"2024-11-21T17:55:20.102224Z","iopub.status.idle":"2024-11-21T17:55:20.108531Z","shell.execute_reply.started":"2024-11-21T17:55:20.102200Z","shell.execute_reply":"2024-11-21T17:55:20.107662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = insulin_corr_spearman[insulin_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:20.109486Z","iopub.execute_input":"2024-11-21T17:55:20.109753Z","iopub.status.idle":"2024-11-21T17:55:20.240167Z","shell.execute_reply.started":"2024-11-21T17:55:20.109725Z","shell.execute_reply":"2024-11-21T17:55:20.239274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insulin= insulin.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:20.241386Z","iopub.execute_input":"2024-11-21T17:55:20.241699Z","iopub.status.idle":"2024-11-21T17:55:20.260484Z","shell.execute_reply.started":"2024-11-21T17:55:20.241672Z","shell.execute_reply":"2024-11-21T17:55:20.259859Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# [dropped features with corr less than average corr]","metadata":{}},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(insulin)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:20.261293Z","iopub.execute_input":"2024-11-21T17:55:20.261519Z","iopub.status.idle":"2024-11-21T17:55:21.516039Z","shell.execute_reply.started":"2024-11-21T17:55:20.261495Z","shell.execute_reply":"2024-11-21T17:55:21.515201Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# steps and target","metadata":{}},{"cell_type":"code","source":"print(f'average skewness in steps cols are {np.mean(steps.skew())}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:21.517192Z","iopub.execute_input":"2024-11-21T17:55:21.517499Z","iopub.status.idle":"2024-11-21T17:55:21.666978Z","shell.execute_reply.started":"2024-11-21T17:55:21.517470Z","shell.execute_reply":"2024-11-21T17:55:21.666268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(steps['steps-0:20'],target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:21.668067Z","iopub.execute_input":"2024-11-21T17:55:21.668413Z","iopub.status.idle":"2024-11-21T17:55:22.105877Z","shell.execute_reply.started":"2024-11-21T17:55:21.668379Z","shell.execute_reply":"2024-11-21T17:55:22.105197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship between `steps` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewness in `steps` feature3.56 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot between `steps['steps-0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between steps columns and the target\nsteps_corr_spearman = steps.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(steps_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:22.106847Z","iopub.execute_input":"2024-11-21T17:55:22.107115Z","iopub.status.idle":"2024-11-21T17:55:22.916530Z","shell.execute_reply.started":"2024-11-21T17:55:22.107087Z","shell.execute_reply":"2024-11-21T17:55:22.915761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(steps_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:22.917624Z","iopub.execute_input":"2024-11-21T17:55:22.918220Z","iopub.status.idle":"2024-11-21T17:55:22.922156Z","shell.execute_reply.started":"2024-11-21T17:55:22.918185Z","shell.execute_reply":"2024-11-21T17:55:22.921504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsteps_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:22.923086Z","iopub.execute_input":"2024-11-21T17:55:22.923379Z","iopub.status.idle":"2024-11-21T17:55:23.459790Z","shell.execute_reply.started":"2024-11-21T17:55:22.923348Z","shell.execute_reply":"2024-11-21T17:55:23.459049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(steps_corr_spearman)\nprint(steps_corr_spearman[steps_corr_spearman < mean_corr])\nprint(f'steps features with corr less than avg: {len(steps_corr_spearman[steps_corr_spearman < mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:23.460769Z","iopub.execute_input":"2024-11-21T17:55:23.461024Z","iopub.status.idle":"2024-11-21T17:55:23.466499Z","shell.execute_reply.started":"2024-11-21T17:55:23.460999Z","shell.execute_reply":"2024-11-21T17:55:23.465852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = steps_corr_spearman[steps_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)\ntest_preprocessed = test_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:23.467318Z","iopub.execute_input":"2024-11-21T17:55:23.467552Z","iopub.status.idle":"2024-11-21T17:55:23.583932Z","shell.execute_reply.started":"2024-11-21T17:55:23.467528Z","shell.execute_reply":"2024-11-21T17:55:23.583185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"steps= steps.drop(columns=features_to_drop.index)\nsteps_test= steps_test.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:23.584904Z","iopub.execute_input":"2024-11-21T17:55:23.585154Z","iopub.status.idle":"2024-11-21T17:55:23.654341Z","shell.execute_reply.started":"2024-11-21T17:55:23.585130Z","shell.execute_reply":"2024-11-21T17:55:23.653636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# [dropped features with corr less than average corr]\r\n","metadata":{}},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(steps)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:23.655214Z","iopub.execute_input":"2024-11-21T17:55:23.655462Z","iopub.status.idle":"2024-11-21T17:55:26.267042Z","shell.execute_reply.started":"2024-11-21T17:55:23.655439Z","shell.execute_reply":"2024-11-21T17:55:26.266354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preprocessed.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:26.267986Z","iopub.execute_input":"2024-11-21T17:55:26.268282Z","iopub.status.idle":"2024-11-21T17:55:26.286079Z","shell.execute_reply.started":"2024-11-21T17:55:26.268215Z","shell.execute_reply":"2024-11-21T17:55:26.285419Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering pipeline","metadata":{}},{"cell_type":"code","source":"def parse_time_column(df, column_name, new_column_name=\"time_column\"):\n   \n    # Remove the word 'time' and strip extra spaces from the original column\n    df[column_name] = df[column_name].str.replace('time', '', regex=False).str.strip()\n\n    # Parse the cleaned-up time strings into datetime objects (only time)\n    df[new_column_name] = pd.to_datetime(df[column_name], format='%H:%M:%S', errors='coerce').dt.strftime('%H:%M:%S')\n    \n    # Drop the original time column\n    df.drop(columns=[column_name], inplace=True)\n\n    return df\n\n# Call the function\ntrain_preprocessed = parse_time_column(train_preprocessed, 'time')\ntest_preprocessed = parse_time_column(test_preprocessed, 'time')\n\n\n# Show the result\nprint(train_preprocessed)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:26.287011Z","iopub.execute_input":"2024-11-21T17:55:26.287265Z","iopub.status.idle":"2024-11-21T17:55:27.367271Z","shell.execute_reply.started":"2024-11-21T17:55:26.287225Z","shell.execute_reply":"2024-11-21T17:55:27.366577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preprocessed = pd.DataFrame(train_preprocessed, columns=train_preprocessed.columns)\n\ntrain_preprocessed.to_csv('train_preprocessed.csv', index=False)\n\ntest_preprocessed = pd.DataFrame(test_preprocessed, columns=test_preprocessed.columns)\n\ntest_preprocessed.to_csv('test_preprocessed.csv', index=False)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:27.368213Z","iopub.execute_input":"2024-11-21T17:55:27.368538Z","iopub.status.idle":"2024-11-21T17:55:48.334966Z","shell.execute_reply.started":"2024-11-21T17:55:27.368506Z","shell.execute_reply":"2024-11-21T17:55:48.334159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\ntrain_preprocessed_loaded = pd.read_csv('train_preprocessed.csv')\ntest_preprocessed_loaded = pd.read_csv('test_preprocessed.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:48.335975Z","iopub.execute_input":"2024-11-21T17:55:48.336345Z","iopub.status.idle":"2024-11-21T17:55:51.615885Z","shell.execute_reply.started":"2024-11-21T17:55:48.336317Z","shell.execute_reply":"2024-11-21T17:55:51.615078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import train_test_split\n\ntrain_preprocessed_loaded.drop(columns=['id','p_num'],inplace=True)\n\n# Defining feature columns and target\nx = train_preprocessed_loaded.drop(columns=['bg+1:00'])\ny = train_preprocessed_loaded['bg+1:00']\n\n# Split the data into training and testing sets\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)\n\n# Select numerical and categorical features\nnumeric_features = x.select_dtypes(include=['int64', 'float64']).columns.tolist()\ncat_features = x.select_dtypes(include='object').columns.tolist()  # Use 'x' instead of 'x_train'\n\n# Define the numerical pipeline (impute missing values + apply power transformation)\nnumerical_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy='median')),            \n    ('power', PowerTransformer(method='yeo-johnson')) \n])\n\n# Create the ColumnTransformer to apply the numerical pipeline to the numeric columns\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numerical_pipeline, numeric_features),\n    ]\n)\n\n# Fit the preprocessor on the training data and then transform the training and test data\nx_train_prep = preprocessor.fit_transform(x_train)  # Fit and transform on training data\nx_test_prep = preprocessor.transform(x_test)       # Only transform on test data\n\n# You can check the transformed data\nprint(x_train_prep)\nprint(x_test_prep)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:55:51.617024Z","iopub.execute_input":"2024-11-21T17:55:51.617322Z","iopub.status.idle":"2024-11-21T17:56:10.301059Z","shell.execute_reply.started":"2024-11-21T17:55:51.617294Z","shell.execute_reply":"2024-11-21T17:56:10.300353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install xgboost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:56:10.302024Z","iopub.execute_input":"2024-11-21T17:56:10.302317Z","iopub.status.idle":"2024-11-21T17:56:14.012122Z","shell.execute_reply.started":"2024-11-21T17:56:10.302288Z","shell.execute_reply":"2024-11-21T17:56:14.011203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import mean_squared_error\n# Initialize the XGBoost Regressor\nxgb_reg = xgb.XGBRegressor(\n    objective='reg:squarederror',  # Objective function for regression\n    n_estimators=100,             # Number of trees\n    learning_rate=0.1,            # Step size shrinkage\n    max_depth=6,                  # Maximum depth of a tree\n    random_state=42               # Ensures reproducibility\n)\n\n# Fit the model\nxgb_reg.fit(x_train_prep, y_train)\n\n# Predict on the test set\ny_pred = xgb_reg.predict(x_test_prep)\n\n# Evaluate\nrmse = mean_squared_error(y_test, y_pred, squared=False)\nprint(f\"RMSE: {rmse}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:56:14.013649Z","iopub.execute_input":"2024-11-21T17:56:14.013974Z","iopub.status.idle":"2024-11-21T17:56:17.256528Z","shell.execute_reply.started":"2024-11-21T17:56:14.013941Z","shell.execute_reply":"2024-11-21T17:56:17.254290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install lightgbm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:56:17.258047Z","iopub.execute_input":"2024-11-21T17:56:17.259965Z","iopub.status.idle":"2024-11-21T17:56:20.843377Z","shell.execute_reply.started":"2024-11-21T17:56:17.259923Z","shell.execute_reply":"2024-11-21T17:56:20.842277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\n\n# Initialize the LightGBM Regressor\nlgb_reg = lgb.LGBMRegressor(\n    objective='regression',       # Objective function for regression\n    n_estimators=100,             # Number of trees\n    learning_rate=0.1,            # Step size shrinkage\n    max_depth=6,                  # Maximum depth of a tree\n    random_state=42               # Ensures reproducibility\n)\n\n# Fit the model\nlgb_reg.fit(x_train_prep, y_train)\n\n# Predict on the test set\ny_pred = lgb_reg.predict(x_test_prep)\n\n# Evaluate\nrmse = mean_squared_error(y_test, y_pred, squared=False)\nprint(f\"RMSE: {rmse}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:56:20.845014Z","iopub.execute_input":"2024-11-21T17:56:20.845361Z","iopub.status.idle":"2024-11-21T17:56:23.460170Z","shell.execute_reply.started":"2024-11-21T17:56:20.845320Z","shell.execute_reply":"2024-11-21T17:56:23.459279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install optuna","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:56:23.461441Z","iopub.execute_input":"2024-11-21T17:56:23.461764Z","iopub.status.idle":"2024-11-21T17:56:27.134646Z","shell.execute_reply.started":"2024-11-21T17:56:23.461731Z","shell.execute_reply":"2024-11-21T17:56:27.133625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\n\ndef objective(trial):\n    # Define hyperparameters to tune\n    params = {\n        'objective': 'reg:squarederror',\n        'n_estimators': trial.suggest_int('n_estimators', 50, 500),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n        'max_depth': trial.suggest_int('max_depth', 3, 10),\n        'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n        'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 1.0, log=True),\n        'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 1.0, log=True),\n    }\n    \n    # Train the model with these parameters\n    model = xgb.XGBRegressor(**params)\n    model.fit(x_train_prep, y_train)\n    \n    # Predict on the validation set\n    y_pred_val = model.predict(x_test_prep)\n    \n    # Return RMSE as the metric to minimize\n    return mean_squared_error(y_test, y_pred_val, squared=False)\n\n# Run Optuna optimization\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\n\n# Best hyperparameters\nprint(\"Best Hyperparameters:\", study.best_params)\nbest_params = study.best_params\nfinal_model = xgb.XGBRegressor(**best_params)\nfinal_model.fit(x_train_prep, y_train)\n\n# Evaluate the final model\ny_pred_final = final_model.predict(x_test_prep)\nfinal_rmse = mean_squared_error(y_test, y_pred_final, squared=False)\nprint(f\"Final RMSE: {final_rmse}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T17:56:27.146067Z","iopub.execute_input":"2024-11-21T17:56:27.146379Z","iopub.status.idle":"2024-11-21T18:15:49.950466Z","shell.execute_reply.started":"2024-11-21T17:56:27.146349Z","shell.execute_reply":"2024-11-21T18:15:49.949359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import  r2_score\n\nr2 = r2_score(y_test,y_pred_final)\n\nprint(f\"R2 Score (rounded predictions): {r2:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:15:49.952012Z","iopub.execute_input":"2024-11-21T18:15:49.952776Z","iopub.status.idle":"2024-11-21T18:15:49.959990Z","shell.execute_reply.started":"2024-11-21T18:15:49.952737Z","shell.execute_reply":"2024-11-21T18:15:49.959289Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preprocessed = preprocessor.fit_transform(test_preprocessed)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:15:49.960960Z","iopub.execute_input":"2024-11-21T18:15:49.961272Z","iopub.status.idle":"2024-11-21T18:15:50.761269Z","shell.execute_reply.started":"2024-11-21T18:15:49.961226Z","shell.execute_reply":"2024-11-21T18:15:50.760385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bg = final_model.predict(test_preprocessed)\nprint(len(bg))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:15:50.762448Z","iopub.execute_input":"2024-11-21T18:15:50.762745Z","iopub.status.idle":"2024-11-21T18:15:50.812814Z","shell.execute_reply.started":"2024-11-21T18:15:50.762716Z","shell.execute_reply":"2024-11-21T18:15:50.811839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.DataFrame()\nt = pd.read_csv('/kaggle/input/brist1d/test.csv')\ndf['id'] = t['id']\ndf['bg+1:00'] = bg\ndf.to_csv('submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:15:50.814446Z","iopub.execute_input":"2024-11-21T18:15:50.814790Z","iopub.status.idle":"2024-11-21T18:15:51.011109Z","shell.execute_reply.started":"2024-11-21T18:15:50.814756Z","shell.execute_reply":"2024-11-21T18:15:51.010308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:15:51.012220Z","iopub.execute_input":"2024-11-21T18:15:51.012533Z","iopub.status.idle":"2024-11-21T18:15:51.023278Z","shell.execute_reply.started":"2024-11-21T18:15:51.012503Z","shell.execute_reply":"2024-11-21T18:15:51.022413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preprocessed = preprocessor.fit_transform(train_preprocessed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:15:51.024526Z","iopub.execute_input":"2024-11-21T18:15:51.024812Z","iopub.status.idle":"2024-11-21T18:16:12.872665Z","shell.execute_reply.started":"2024-11-21T18:15:51.024785Z","shell.execute_reply":"2024-11-21T18:16:12.871725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bg = final_model.predict(train_preprocessed)\nprint(len(bg))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:16:12.873960Z","iopub.execute_input":"2024-11-21T18:16:12.874307Z","iopub.status.idle":"2024-11-21T18:16:12.985186Z","shell.execute_reply.started":"2024-11-21T18:16:12.874275Z","shell.execute_reply":"2024-11-21T18:16:12.984304Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df2 = pd.DataFrame()\nt = pd.read_csv('/kaggle/input/brist1d/train.csv')\ndf2['id'] = t['id']\ndf2['bg+1:00'] = bg\ndf2.to_csv('submission2.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:16:12.986626Z","iopub.execute_input":"2024-11-21T18:16:12.987181Z","iopub.status.idle":"2024-11-21T18:16:21.010838Z","shell.execute_reply.started":"2024-11-21T18:16:12.987142Z","shell.execute_reply":"2024-11-21T18:16:21.009815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T18:16:21.011962Z","iopub.execute_input":"2024-11-21T18:16:21.012229Z","iopub.status.idle":"2024-11-21T18:16:21.022256Z","shell.execute_reply.started":"2024-11-21T18:16:21.012204Z","shell.execute_reply":"2024-11-21T18:16:21.021205Z"}},"outputs":[],"execution_count":null}]}