{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:13.156679Z","iopub.execute_input":"2024-11-22T07:39:13.157064Z","iopub.status.idle":"2024-11-22T07:39:13.160824Z","shell.execute_reply.started":"2024-11-22T07:39:13.157032Z","shell.execute_reply":"2024-11-22T07:39:13.160205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/brist1d/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/brist1d/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:13.163888Z","iopub.execute_input":"2024-11-22T07:39:13.164129Z","iopub.status.idle":"2024-11-22T07:39:20.974830Z","shell.execute_reply.started":"2024-11-22T07:39:13.164105Z","shell.execute_reply":"2024-11-22T07:39:20.973939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Shape of train ',train.shape)\nprint(f'shape of test {test.shape} ')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:20.976378Z","iopub.execute_input":"2024-11-22T07:39:20.976678Z","iopub.status.idle":"2024-11-22T07:39:20.980947Z","shell.execute_reply.started":"2024-11-22T07:39:20.976649Z","shell.execute_reply":"2024-11-22T07:39:20.980209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.sample(4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:20.982638Z","iopub.execute_input":"2024-11-22T07:39:20.982887Z","iopub.status.idle":"2024-11-22T07:39:21.012932Z","shell.execute_reply.started":"2024-11-22T07:39:20.982862Z","shell.execute_reply":"2024-11-22T07:39:21.012129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.sample(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:21.014765Z","iopub.execute_input":"2024-11-22T07:39:21.015054Z","iopub.status.idle":"2024-11-22T07:39:21.034497Z","shell.execute_reply.started":"2024-11-22T07:39:21.015025Z","shell.execute_reply":"2024-11-22T07:39:21.033778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info(show_counts=True,verbose = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:21.035459Z","iopub.execute_input":"2024-11-22T07:39:21.035760Z","iopub.status.idle":"2024-11-22T07:39:21.583008Z","shell.execute_reply.started":"2024-11-22T07:39:21.035729Z","shell.execute_reply":"2024-11-22T07:39:21.582319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols=[]\nfor i in train.columns:\n    col = i.split('-')[0]\n    uniq_cols.append(col)\nuniq_cols = np.unique(uniq_cols)\nprint(uniq_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:21.583954Z","iopub.execute_input":"2024-11-22T07:39:21.584225Z","iopub.status.idle":"2024-11-22T07:39:21.589588Z","shell.execute_reply.started":"2024-11-22T07:39:21.584199Z","shell.execute_reply":"2024-11-22T07:39:21.588907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols=[]\nfor i in train.columns:\n    col = i.split('-')[0]\n    uniq_cols.append(col)\nuniq_cols = np.unique(uniq_cols)\nprint(uniq_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:21.590479Z","iopub.execute_input":"2024-11-22T07:39:21.590747Z","iopub.status.idle":"2024-11-22T07:39:21.611488Z","shell.execute_reply.started":"2024-11-22T07:39:21.590719Z","shell.execute_reply":"2024-11-22T07:39:21.610817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols=[]\nfor i in test.columns:\n    col = i.split('-')[0]\n    uniq_cols.append(col)\nuniq_cols = np.unique(uniq_cols)\nprint(uniq_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:21.612310Z","iopub.execute_input":"2024-11-22T07:39:21.612534Z","iopub.status.idle":"2024-11-22T07:39:21.631296Z","shell.execute_reply.started":"2024-11-22T07:39:21.612510Z","shell.execute_reply":"2024-11-22T07:39:21.630716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tabulate import tabulate\nnulls_train = train.isnull().sum()\nnull_percent_train = train.isnull().mean() * 100\n# Prepare data for the table\ntable_data = list(zip(nulls_train.index, nulls_train.values, null_percent_train.values))\nheaders = ['Column', 'Null Count', 'Null Percentage']\ntable = tabulate(table_data, headers, tablefmt='pretty')\nprint(table)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:21.632154Z","iopub.execute_input":"2024-11-22T07:39:21.632406Z","iopub.status.idle":"2024-11-22T07:39:22.591234Z","shell.execute_reply.started":"2024-11-22T07:39:21.632381Z","shell.execute_reply":"2024-11-22T07:39:22.590549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tabulate import tabulate\nnulls_test = test.isnull().sum()\nnull_percent_test = test.isnull().mean() * 100\n# Prepare data for the table\ntable_data = list(zip(nulls_test.index, nulls_test.values, null_percent_test.values))\nheaders = ['Column', 'Null Count', 'Null Percentage']\ntable = tabulate(table_data, headers, tablefmt='pretty')\nprint(table)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.593773Z","iopub.execute_input":"2024-11-22T07:39:22.594027Z","iopub.status.idle":"2024-11-22T07:39:22.641928Z","shell.execute_reply.started":"2024-11-22T07:39:22.594003Z","shell.execute_reply":"2024-11-22T07:39:22.641314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_percent_train.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.642701Z","iopub.execute_input":"2024-11-22T07:39:22.642928Z","iopub.status.idle":"2024-11-22T07:39:22.647544Z","shell.execute_reply.started":"2024-11-22T07:39:22.642905Z","shell.execute_reply":"2024-11-22T07:39:22.646860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_percent_test.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.648555Z","iopub.execute_input":"2024-11-22T07:39:22.648987Z","iopub.status.idle":"2024-11-22T07:39:22.658086Z","shell.execute_reply.started":"2024-11-22T07:39:22.648957Z","shell.execute_reply":"2024-11-22T07:39:22.657386Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## On average 44% values are mising from data","metadata":{}},{"cell_type":"markdown","source":"### On each unique col I am making a separate df to check each features relation with target then startegy for missing values will be developed","metadata":{}},{"cell_type":"code","source":"uniq_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.658962Z","iopub.execute_input":"2024-11-22T07:39:22.659251Z","iopub.status.idle":"2024-11-22T07:39:22.668686Z","shell.execute_reply.started":"2024-11-22T07:39:22.659225Z","shell.execute_reply":"2024-11-22T07:39:22.668023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.669462Z","iopub.execute_input":"2024-11-22T07:39:22.669694Z","iopub.status.idle":"2024-11-22T07:39:22.679152Z","shell.execute_reply.started":"2024-11-22T07:39:22.669671Z","shell.execute_reply":"2024-11-22T07:39:22.678568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## this function will make a separate df for each col see the result to understand\ndef df_of_col (train,col):\n    df = pd.DataFrame()\n    for i in train.columns:\n        if (i.split('-')[0] ==col) & (i  not in ['hr','bg+1:00','id','p_num']):\n            df[i] = train[i]\n    return df\ndef df_of_col (test,col):\n    df = pd.DataFrame()\n    for i in train.columns:\n        if (i.split('-')[0] ==col) & (i  not in ['hr','bg+1:00','id','p_num']):\n            df[i] = test[i]\n    return df\n       ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.679959Z","iopub.execute_input":"2024-11-22T07:39:22.680206Z","iopub.status.idle":"2024-11-22T07:39:22.690141Z","shell.execute_reply.started":"2024-11-22T07:39:22.680181Z","shell.execute_reply":"2024-11-22T07:39:22.689569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"activity_df = df_of_col(train,'activity')\nactivity_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.690871Z","iopub.execute_input":"2024-11-22T07:39:22.691080Z","iopub.status.idle":"2024-11-22T07:39:22.788005Z","shell.execute_reply.started":"2024-11-22T07:39:22.691058Z","shell.execute_reply":"2024-11-22T07:39:22.787215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.788987Z","iopub.execute_input":"2024-11-22T07:39:22.789245Z","iopub.status.idle":"2024-11-22T07:39:22.793764Z","shell.execute_reply.started":"2024-11-22T07:39:22.789220Z","shell.execute_reply":"2024-11-22T07:39:22.793152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Train\nactivity = df_of_col(train,uniq_cols[0])\nbg =  df_of_col(train,uniq_cols[1])\ncals =  df_of_col(train,uniq_cols[2])\ncarbs =  df_of_col(train,uniq_cols[3])\ninsulin =  df_of_col(train,'insulin')\nsteps =  df_of_col(train,uniq_cols[-2])\ntime =  df_of_col(train,uniq_cols[-1])\n#Test\nactivity = df_of_col(test,uniq_cols[0])\nbg_test =  df_of_col(test,uniq_cols[1])\ncals_test =  df_of_col(test,uniq_cols[2])\ncarbs_test =  df_of_col(test,uniq_cols[3])\ninsulin_test =  df_of_col(test,'insulin')\nsteps_test =  df_of_col(test,uniq_cols[-2])\ntime_test =  df_of_col(test,uniq_cols[-1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:22.794624Z","iopub.execute_input":"2024-11-22T07:39:22.794878Z","iopub.status.idle":"2024-11-22T07:39:23.232648Z","shell.execute_reply.started":"2024-11-22T07:39:22.794852Z","shell.execute_reply":"2024-11-22T07:39:23.231906Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" ## target feature's distribution","metadata":{}},{"cell_type":"code","source":"target = train['bg+1:00']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:23.233619Z","iopub.execute_input":"2024-11-22T07:39:23.233873Z","iopub.status.idle":"2024-11-22T07:39:23.239212Z","shell.execute_reply.started":"2024-11-22T07:39:23.233848Z","shell.execute_reply":"2024-11-22T07:39:23.238580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.distplot(target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:23.240018Z","iopub.execute_input":"2024-11-22T07:39:23.240268Z","iopub.status.idle":"2024-11-22T07:39:24.145855Z","shell.execute_reply.started":"2024-11-22T07:39:23.240244Z","shell.execute_reply":"2024-11-22T07:39:24.145206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'skewness in bg+1:00 is {target.skew()}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:24.146865Z","iopub.execute_input":"2024-11-22T07:39:24.147199Z","iopub.status.idle":"2024-11-22T07:39:24.152790Z","shell.execute_reply.started":"2024-11-22T07:39:24.147146Z","shell.execute_reply":"2024-11-22T07:39:24.152107Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## activity features and target","metadata":{}},{"cell_type":"code","source":"activity_missing_percentage = activity.isnull().mean() * 100\nprint(activity_missing_percentage)\nprint(\"------------------------------------------------------------------------------\")\nprint(f'mean percentage for missing values is {np.mean(activity_missing_percentage)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:24.153734Z","iopub.execute_input":"2024-11-22T07:39:24.154000Z","iopub.status.idle":"2024-11-22T07:39:24.174126Z","shell.execute_reply.started":"2024-11-22T07:39:24.153972Z","shell.execute_reply":"2024-11-22T07:39:24.173500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'on average this much values are missing for activity columns : {np.mean(activity.isnull().sum())}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:24.174925Z","iopub.execute_input":"2024-11-22T07:39:24.175195Z","iopub.status.idle":"2024-11-22T07:39:24.190722Z","shell.execute_reply.started":"2024-11-22T07:39:24.175155Z","shell.execute_reply":"2024-11-22T07:39:24.190063Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### so its better to drop alll columns of activity from training","metadata":{}},{"cell_type":"code","source":"train_preprocessed = train.drop(columns=activity.columns)\ntest_preprocessed = test.drop(columns=activity.columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:24.191605Z","iopub.execute_input":"2024-11-22T07:39:24.191849Z","iopub.status.idle":"2024-11-22T07:39:24.381743Z","shell.execute_reply.started":"2024-11-22T07:39:24.191824Z","shell.execute_reply":"2024-11-22T07:39:24.380969Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## bg and target","metadata":{}},{"cell_type":"code","source":"bg.sample(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:24.382717Z","iopub.execute_input":"2024-11-22T07:39:24.382969Z","iopub.status.idle":"2024-11-22T07:39:24.416622Z","shell.execute_reply.started":"2024-11-22T07:39:24.382945Z","shell.execute_reply":"2024-11-22T07:39:24.415961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average skewness in bg cols are {np.mean(bg.skew())}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:24.417577Z","iopub.execute_input":"2024-11-22T07:39:24.417856Z","iopub.status.idle":"2024-11-22T07:39:24.521897Z","shell.execute_reply.started":"2024-11-22T07:39:24.417826Z","shell.execute_reply":"2024-11-22T07:39:24.521210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(bg['bg-0:20'],target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:24.522846Z","iopub.execute_input":"2024-11-22T07:39:24.523129Z","iopub.status.idle":"2024-11-22T07:39:25.181914Z","shell.execute_reply.started":"2024-11-22T07:39:24.523101Z","shell.execute_reply":"2024-11-22T07:39:25.181210Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship between `bg` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewness in `bg` features is 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot between `bg['bg+0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between bg columns and the target\nbg_corr_spearman = bg.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(bg_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:25.187124Z","iopub.execute_input":"2024-11-22T07:39:25.187433Z","iopub.status.idle":"2024-11-22T07:39:26.244804Z","shell.execute_reply.started":"2024-11-22T07:39:25.187404Z","shell.execute_reply":"2024-11-22T07:39:26.244093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(bg_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:26.245707Z","iopub.execute_input":"2024-11-22T07:39:26.245950Z","iopub.status.idle":"2024-11-22T07:39:26.249976Z","shell.execute_reply.started":"2024-11-22T07:39:26.245925Z","shell.execute_reply":"2024-11-22T07:39:26.249349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nbg_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:26.250745Z","iopub.execute_input":"2024-11-22T07:39:26.250961Z","iopub.status.idle":"2024-11-22T07:39:26.750846Z","shell.execute_reply.started":"2024-11-22T07:39:26.250938Z","shell.execute_reply":"2024-11-22T07:39:26.750054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(bg_corr_spearman)\nprint(bg_corr_spearman[bg_corr_spearman < mean_corr])\nprint(f'bg features with corr less than avg: {len(bg_corr_spearman[bg_corr_spearman < mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:26.751861Z","iopub.execute_input":"2024-11-22T07:39:26.752127Z","iopub.status.idle":"2024-11-22T07:39:26.758732Z","shell.execute_reply.started":"2024-11-22T07:39:26.752101Z","shell.execute_reply":"2024-11-22T07:39:26.757962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = bg_corr_spearman[bg_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)\ntest_preprocessed = test_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:26.759675Z","iopub.execute_input":"2024-11-22T07:39:26.760060Z","iopub.status.idle":"2024-11-22T07:39:26.945080Z","shell.execute_reply.started":"2024-11-22T07:39:26.760032Z","shell.execute_reply":"2024-11-22T07:39:26.944192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bg= bg.drop(columns=features_to_drop.index)\nbg_test= bg_test.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:26.946110Z","iopub.execute_input":"2024-11-22T07:39:26.946412Z","iopub.status.idle":"2024-11-22T07:39:26.966297Z","shell.execute_reply.started":"2024-11-22T07:39:26.946383Z","shell.execute_reply":"2024-11-22T07:39:26.965541Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### dropped features with corr less than average corr","metadata":{}},{"cell_type":"code","source":"pip install missingno","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:26.967328Z","iopub.execute_input":"2024-11-22T07:39:26.967636Z","iopub.status.idle":"2024-11-22T07:39:30.530824Z","shell.execute_reply.started":"2024-11-22T07:39:26.967605Z","shell.execute_reply":"2024-11-22T07:39:30.530043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(bg)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:30.532319Z","iopub.execute_input":"2024-11-22T07:39:30.532611Z","iopub.status.idle":"2024-11-22T07:39:32.036759Z","shell.execute_reply.started":"2024-11-22T07:39:30.532583Z","shell.execute_reply":"2024-11-22T07:39:32.035985Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## cals and target","metadata":{}},{"cell_type":"code","source":"cals.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:32.037735Z","iopub.execute_input":"2024-11-22T07:39:32.038013Z","iopub.status.idle":"2024-11-22T07:39:32.065801Z","shell.execute_reply.started":"2024-11-22T07:39:32.037985Z","shell.execute_reply":"2024-11-22T07:39:32.065211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average skewness in calories cols are {np.mean(cals.skew())}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:32.066633Z","iopub.execute_input":"2024-11-22T07:39:32.066859Z","iopub.status.idle":"2024-11-22T07:39:32.175691Z","shell.execute_reply.started":"2024-11-22T07:39:32.066836Z","shell.execute_reply":"2024-11-22T07:39:32.174966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(cals['cals-0:20'],target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:32.176677Z","iopub.execute_input":"2024-11-22T07:39:32.176936Z","iopub.status.idle":"2024-11-22T07:39:32.820775Z","shell.execute_reply.started":"2024-11-22T07:39:32.176911Z","shell.execute_reply":"2024-11-22T07:39:32.820092Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship betweencalsbg` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewnesscalsn `bg` feature2.73 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot bcalswecals-`bg['bg+0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between cals columns and the target\ncals_corr_spearman = cals.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(cals_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:32.821739Z","iopub.execute_input":"2024-11-22T07:39:32.821991Z","iopub.status.idle":"2024-11-22T07:39:34.259349Z","shell.execute_reply.started":"2024-11-22T07:39:32.821965Z","shell.execute_reply":"2024-11-22T07:39:34.258541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(cals_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:34.260360Z","iopub.execute_input":"2024-11-22T07:39:34.260621Z","iopub.status.idle":"2024-11-22T07:39:34.264389Z","shell.execute_reply.started":"2024-11-22T07:39:34.260596Z","shell.execute_reply":"2024-11-22T07:39:34.263769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ncals_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:34.265219Z","iopub.execute_input":"2024-11-22T07:39:34.265455Z","iopub.status.idle":"2024-11-22T07:39:34.805919Z","shell.execute_reply.started":"2024-11-22T07:39:34.265431Z","shell.execute_reply":"2024-11-22T07:39:34.804973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(cals_corr_spearman)\nprint(cals_corr_spearman[cals_corr_spearman < mean_corr])\nprint(f'cals features with corr less than avg: {len(cals_corr_spearman[cals_corr_spearman < mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:34.807098Z","iopub.execute_input":"2024-11-22T07:39:34.807396Z","iopub.status.idle":"2024-11-22T07:39:34.813743Z","shell.execute_reply.started":"2024-11-22T07:39:34.807367Z","shell.execute_reply":"2024-11-22T07:39:34.813075Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = cals_corr_spearman[cals_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)\ntest_preprocessed = test_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:34.814695Z","iopub.execute_input":"2024-11-22T07:39:34.814949Z","iopub.status.idle":"2024-11-22T07:39:34.993277Z","shell.execute_reply.started":"2024-11-22T07:39:34.814923Z","shell.execute_reply":"2024-11-22T07:39:34.992503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cals= cals.drop(columns=features_to_drop.index)\ncals_test= cals_test.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:34.994329Z","iopub.execute_input":"2024-11-22T07:39:34.994626Z","iopub.status.idle":"2024-11-22T07:39:35.030889Z","shell.execute_reply.started":"2024-11-22T07:39:34.994598Z","shell.execute_reply":"2024-11-22T07:39:35.030204Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cals","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:35.032090Z","iopub.execute_input":"2024-11-22T07:39:35.032400Z","iopub.status.idle":"2024-11-22T07:39:35.155402Z","shell.execute_reply.started":"2024-11-22T07:39:35.032370Z","shell.execute_reply":"2024-11-22T07:39:35.154765Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### dropped features with corr less than average corr","metadata":{}},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(cals)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:35.156580Z","iopub.execute_input":"2024-11-22T07:39:35.156871Z","iopub.status.idle":"2024-11-22T07:39:37.421871Z","shell.execute_reply.started":"2024-11-22T07:39:35.156845Z","shell.execute_reply":"2024-11-22T07:39:37.421207Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# carbs and target","metadata":{}},{"cell_type":"code","source":"print(f'average skewness in carbs cols are {np.mean(carbs.skew())}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:37.422835Z","iopub.execute_input":"2024-11-22T07:39:37.423087Z","iopub.status.idle":"2024-11-22T07:39:37.540677Z","shell.execute_reply.started":"2024-11-22T07:39:37.423063Z","shell.execute_reply":"2024-11-22T07:39:37.539962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(carbs['carbs-1:40'],target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:37.541693Z","iopub.execute_input":"2024-11-22T07:39:37.541970Z","iopub.status.idle":"2024-11-22T07:39:37.677550Z","shell.execute_reply.started":"2024-11-22T07:39:37.541943Z","shell.execute_reply":"2024-11-22T07:39:37.676899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship between `carbs` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewness in `carbs` feature6.83 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot between `carb-['carbs+0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between carbs columns and the target\ncarbs_corr_spearman = carbs.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(carbs_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:37.678655Z","iopub.execute_input":"2024-11-22T07:39:37.678965Z","iopub.status.idle":"2024-11-22T07:39:37.853791Z","shell.execute_reply.started":"2024-11-22T07:39:37.678934Z","shell.execute_reply":"2024-11-22T07:39:37.853148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(carbs_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:37.854689Z","iopub.execute_input":"2024-11-22T07:39:37.854934Z","iopub.status.idle":"2024-11-22T07:39:37.858746Z","shell.execute_reply.started":"2024-11-22T07:39:37.854910Z","shell.execute_reply":"2024-11-22T07:39:37.858057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ncarbs_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:37.859618Z","iopub.execute_input":"2024-11-22T07:39:37.859857Z","iopub.status.idle":"2024-11-22T07:39:38.371285Z","shell.execute_reply.started":"2024-11-22T07:39:37.859834Z","shell.execute_reply":"2024-11-22T07:39:38.370651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(carbs_corr_spearman)\nprint(carbs_corr_spearman[carbs_corr_spearman < mean_corr])\nprint(f'carbs features with corr less than avg: {len(carbs_corr_spearman[carbs_corr_spearman < mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:38.372189Z","iopub.execute_input":"2024-11-22T07:39:38.372432Z","iopub.status.idle":"2024-11-22T07:39:38.378027Z","shell.execute_reply.started":"2024-11-22T07:39:38.372408Z","shell.execute_reply":"2024-11-22T07:39:38.377351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = carbs_corr_spearman[carbs_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)\ntest_preprocessed = test_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:38.378919Z","iopub.execute_input":"2024-11-22T07:39:38.379184Z","iopub.status.idle":"2024-11-22T07:39:38.531461Z","shell.execute_reply.started":"2024-11-22T07:39:38.379143Z","shell.execute_reply":"2024-11-22T07:39:38.530649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"carbs= carbs.drop(columns=features_to_drop.index)\ncarbs_test= carbs_test.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:38.532588Z","iopub.execute_input":"2024-11-22T07:39:38.532884Z","iopub.status.idle":"2024-11-22T07:39:38.561919Z","shell.execute_reply.started":"2024-11-22T07:39:38.532855Z","shell.execute_reply":"2024-11-22T07:39:38.561219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(carbs)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:38.562971Z","iopub.execute_input":"2024-11-22T07:39:38.563286Z","iopub.status.idle":"2024-11-22T07:39:40.328756Z","shell.execute_reply.started":"2024-11-22T07:39:38.563258Z","shell.execute_reply":"2024-11-22T07:39:40.327994Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# insulin and target","metadata":{}},{"cell_type":"code","source":"print(f'average skewness in insulin cols are {np.mean(insulin.skew())}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:40.329773Z","iopub.execute_input":"2024-11-22T07:39:40.330045Z","iopub.status.idle":"2024-11-22T07:39:40.432629Z","shell.execute_reply.started":"2024-11-22T07:39:40.330020Z","shell.execute_reply":"2024-11-22T07:39:40.431879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(insulin['insulin-0:20'],target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:40.433701Z","iopub.execute_input":"2024-11-22T07:39:40.433975Z","iopub.status.idle":"2024-11-22T07:39:41.174414Z","shell.execute_reply.started":"2024-11-22T07:39:40.433948Z","shell.execute_reply":"2024-11-22T07:39:41.173697Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship between `insulin` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewness in `insulin` features4 52 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot between `insulin['insulin-0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between insulin columns and the target\ninsulin_corr_spearman = insulin.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(insulin_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:41.175532Z","iopub.execute_input":"2024-11-22T07:39:41.175862Z","iopub.status.idle":"2024-11-22T07:39:42.860342Z","shell.execute_reply.started":"2024-11-22T07:39:41.175831Z","shell.execute_reply":"2024-11-22T07:39:42.859592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(insulin_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:42.861305Z","iopub.execute_input":"2024-11-22T07:39:42.861568Z","iopub.status.idle":"2024-11-22T07:39:42.865344Z","shell.execute_reply.started":"2024-11-22T07:39:42.861544Z","shell.execute_reply":"2024-11-22T07:39:42.864701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ninsulin_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:42.866254Z","iopub.execute_input":"2024-11-22T07:39:42.866510Z","iopub.status.idle":"2024-11-22T07:39:43.375740Z","shell.execute_reply.started":"2024-11-22T07:39:42.866475Z","shell.execute_reply":"2024-11-22T07:39:43.375110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(insulin_corr_spearman)\nprint(insulin_corr_spearman[insulin_corr_spearman < mean_corr])\nprint(f'insulin features with corr less than avg: {len(insulin_corr_spearman[insulin_corr_spearman< mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:43.376651Z","iopub.execute_input":"2024-11-22T07:39:43.376891Z","iopub.status.idle":"2024-11-22T07:39:43.383160Z","shell.execute_reply.started":"2024-11-22T07:39:43.376868Z","shell.execute_reply":"2024-11-22T07:39:43.382378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = insulin_corr_spearman[insulin_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:43.384062Z","iopub.execute_input":"2024-11-22T07:39:43.384395Z","iopub.status.idle":"2024-11-22T07:39:43.508605Z","shell.execute_reply.started":"2024-11-22T07:39:43.384370Z","shell.execute_reply":"2024-11-22T07:39:43.507739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"insulin= insulin.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:43.509722Z","iopub.execute_input":"2024-11-22T07:39:43.510014Z","iopub.status.idle":"2024-11-22T07:39:43.526553Z","shell.execute_reply.started":"2024-11-22T07:39:43.509985Z","shell.execute_reply":"2024-11-22T07:39:43.525929Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# [dropped features with corr less than average corr]","metadata":{}},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(insulin)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:43.527401Z","iopub.execute_input":"2024-11-22T07:39:43.527639Z","iopub.status.idle":"2024-11-22T07:39:44.569982Z","shell.execute_reply.started":"2024-11-22T07:39:43.527616Z","shell.execute_reply":"2024-11-22T07:39:44.569222Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# steps and target","metadata":{}},{"cell_type":"code","source":"print(f'average skewness in steps cols are {np.mean(steps.skew())}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:44.570905Z","iopub.execute_input":"2024-11-22T07:39:44.571150Z","iopub.status.idle":"2024-11-22T07:39:44.715999Z","shell.execute_reply.started":"2024-11-22T07:39:44.571126Z","shell.execute_reply":"2024-11-22T07:39:44.715285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.scatter(steps['steps-0:20'],target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:44.716962Z","iopub.execute_input":"2024-11-22T07:39:44.717226Z","iopub.status.idle":"2024-11-22T07:39:45.157156Z","shell.execute_reply.started":"2024-11-22T07:39:44.717201Z","shell.execute_reply":"2024-11-22T07:39:45.156504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose **Spearman correlation** to assess the relationship between `steps` features and the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of 1.07, and the average skewness in `steps` feature3.56 1.06, indicating a **non-normal distribution**. Spearman correlation is more robust to skewed data than Pearson correlation.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplot between `steps['steps-0:20']` and the target shows a **non-linear, monotonic relationship** rather than a clear linear pattern. Spearman correlation captures monotonic relationships better, making it a suitable choice in this context.\r\n","metadata":{}},{"cell_type":"code","source":"# Calculate Spearman correlation between steps columns and the target\nsteps_corr_spearman = steps.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations:\")\nprint(steps_corr_spearman)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:45.158144Z","iopub.execute_input":"2024-11-22T07:39:45.158448Z","iopub.status.idle":"2024-11-22T07:39:46.001895Z","shell.execute_reply.started":"2024-11-22T07:39:45.158419Z","shell.execute_reply":"2024-11-22T07:39:46.001159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'average corr is {np.mean(steps_corr_spearman)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:46.002937Z","iopub.execute_input":"2024-11-22T07:39:46.003245Z","iopub.status.idle":"2024-11-22T07:39:46.007340Z","shell.execute_reply.started":"2024-11-22T07:39:46.003216Z","shell.execute_reply":"2024-11-22T07:39:46.006574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsteps_corr_spearman.plot(kind='bar', color='skyblue', edgecolor='black')\nplt.title(\"Spearman Correlation of Each Feature with Target\")\nplt.xlabel(\"Features\")\nplt.ylabel(\"Spearman Correlation\")\nplt.xticks(rotation=90)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:46.008325Z","iopub.execute_input":"2024-11-22T07:39:46.008626Z","iopub.status.idle":"2024-11-22T07:39:46.540109Z","shell.execute_reply.started":"2024-11-22T07:39:46.008596Z","shell.execute_reply":"2024-11-22T07:39:46.539333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_corr = np.mean(steps_corr_spearman)\nprint(steps_corr_spearman[steps_corr_spearman < mean_corr])\nprint(f'steps features with corr less than avg: {len(steps_corr_spearman[steps_corr_spearman < mean_corr])}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:46.541094Z","iopub.execute_input":"2024-11-22T07:39:46.541360Z","iopub.status.idle":"2024-11-22T07:39:46.547137Z","shell.execute_reply.started":"2024-11-22T07:39:46.541334Z","shell.execute_reply":"2024-11-22T07:39:46.546363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_to_drop = steps_corr_spearman[steps_corr_spearman < mean_corr]\ntrain_preprocessed = train_preprocessed.drop(columns=features_to_drop.index)\ntest_preprocessed = test_preprocessed.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:46.547948Z","iopub.execute_input":"2024-11-22T07:39:46.548210Z","iopub.status.idle":"2024-11-22T07:39:46.666009Z","shell.execute_reply.started":"2024-11-22T07:39:46.548141Z","shell.execute_reply":"2024-11-22T07:39:46.665237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"steps= steps.drop(columns=features_to_drop.index)\nsteps_test= steps_test.drop(columns=features_to_drop.index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:46.667070Z","iopub.execute_input":"2024-11-22T07:39:46.667446Z","iopub.status.idle":"2024-11-22T07:39:46.705376Z","shell.execute_reply.started":"2024-11-22T07:39:46.667417Z","shell.execute_reply":"2024-11-22T07:39:46.704740Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# [dropped features with corr less than average corr]\r\n","metadata":{}},{"cell_type":"code","source":"import missingno as msno\nimport matplotlib.pyplot as plt\n\n# Assuming bg is your DataFrame\nplt.figure(figsize=(10, 5))\n\n# Missingness Heatmap\nmsno.heatmap(steps)\nplt.title(\"Missingness Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:46.706238Z","iopub.execute_input":"2024-11-22T07:39:46.706467Z","iopub.status.idle":"2024-11-22T07:39:49.524358Z","shell.execute_reply.started":"2024-11-22T07:39:46.706445Z","shell.execute_reply":"2024-11-22T07:39:49.523622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preprocessed.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:49.525324Z","iopub.execute_input":"2024-11-22T07:39:49.525834Z","iopub.status.idle":"2024-11-22T07:39:49.543350Z","shell.execute_reply.started":"2024-11-22T07:39:49.525806Z","shell.execute_reply":"2024-11-22T07:39:49.542714Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering pipeline","metadata":{}},{"cell_type":"code","source":"def parse_time_column(df, column_name, new_column_name=\"time_column\"):\n   \n    # Remove the word 'time' and strip extra spaces from the original column\n    df[column_name] = df[column_name].str.replace('time', '', regex=False).str.strip()\n\n    # Parse the cleaned-up time strings into datetime objects (only time)\n    df[new_column_name] = pd.to_datetime(df[column_name], format='%H:%M:%S', errors='coerce').dt.strftime('%H:%M:%S')\n    \n    # Drop the original time column\n    df.drop(columns=[column_name], inplace=True)\n\n    return df\n\n# Call the function\ntrain_preprocessed = parse_time_column(train_preprocessed, 'time')\ntest_preprocessed = parse_time_column(test_preprocessed, 'time')\n\n\n# Show the result\nprint(train_preprocessed)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:49.544189Z","iopub.execute_input":"2024-11-22T07:39:49.544426Z","iopub.status.idle":"2024-11-22T07:39:50.608089Z","shell.execute_reply.started":"2024-11-22T07:39:49.544401Z","shell.execute_reply":"2024-11-22T07:39:50.607390Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preprocessed = pd.DataFrame(train_preprocessed, columns=train_preprocessed.columns)\n\ntrain_preprocessed.to_csv('train_preprocessed.csv', index=False)\n\ntest_preprocessed = pd.DataFrame(test_preprocessed, columns=test_preprocessed.columns)\n\ntest_preprocessed.to_csv('test_preprocessed.csv', index=False)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:39:50.609158Z","iopub.execute_input":"2024-11-22T07:39:50.609498Z","iopub.status.idle":"2024-11-22T07:40:11.569822Z","shell.execute_reply.started":"2024-11-22T07:39:50.609457Z","shell.execute_reply":"2024-11-22T07:40:11.569047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\ntrain_preprocessed_loaded = pd.read_csv('train_preprocessed.csv')\ntest_preprocessed_loaded = pd.read_csv('test_preprocessed.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:40:11.570951Z","iopub.execute_input":"2024-11-22T07:40:11.571252Z","iopub.status.idle":"2024-11-22T07:40:14.496930Z","shell.execute_reply.started":"2024-11-22T07:40:11.571222Z","shell.execute_reply":"2024-11-22T07:40:14.496158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import train_test_split\n\ntrain_preprocessed_loaded.drop(columns=['id','p_num'],inplace=True)\n\n# Defining feature columns and target\nx = train_preprocessed_loaded.drop(columns=['bg+1:00'])\ny = train_preprocessed_loaded['bg+1:00']\n\n# Split the data into training and testing sets\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)\n\n# Select numerical and categorical features\nnumeric_features = x.select_dtypes(include=['int64', 'float64']).columns.tolist()\ncat_features = x.select_dtypes(include='object').columns.tolist()  # Use 'x' instead of 'x_train'\n\n# Define the numerical pipeline (impute missing values + apply power transformation)\nnumerical_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy='median')),            \n    ('power', PowerTransformer(method='yeo-johnson')) \n])\n\n# Create the ColumnTransformer to apply the numerical pipeline to the numeric columns\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numerical_pipeline, numeric_features),\n    ]\n)\n\n# Fit the preprocessor on the training data and then transform the training and test data\nx_train_prep = preprocessor.fit_transform(x_train)  # Fit and transform on training data\nx_test_prep = preprocessor.transform(x_test)       # Only transform on test data\n\n# You can check the transformed data\nprint(x_train_prep)\nprint(x_test_prep)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:40:14.497890Z","iopub.execute_input":"2024-11-22T07:40:14.498136Z","iopub.status.idle":"2024-11-22T07:40:33.689143Z","shell.execute_reply.started":"2024-11-22T07:40:14.498111Z","shell.execute_reply":"2024-11-22T07:40:33.688373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install lightgbm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:40:33.690158Z","iopub.execute_input":"2024-11-22T07:40:33.690447Z","iopub.status.idle":"2024-11-22T07:40:37.851024Z","shell.execute_reply.started":"2024-11-22T07:40:33.690419Z","shell.execute_reply":"2024-11-22T07:40:37.849985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import mean_squared_error\n# Initialize the XGBoost Regressor\nxgb_reg = xgb.XGBRegressor(\n    objective='reg:squarederror',  # Objective function for regression\n    n_estimators=100,             # Number of trees\n    learning_rate=0.1,            # Step size shrinkage\n    max_depth=6,                  # Maximum depth of a tree\n    random_state=42               # Ensures reproducibility\n)\n\n# Fit the model\nxgb_reg.fit(x_train_prep, y_train)\n\n# Predict on the test set\ny_pred = xgb_reg.predict(x_test_prep)\n\n# Evaluate\nrmse = mean_squared_error(y_test, y_pred, squared=False)\nprint(f\"RMSE: {rmse}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install optuna","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T07:40:37.852456Z","iopub.execute_input":"2024-11-22T07:40:37.852815Z","iopub.status.idle":"2024-11-22T07:40:45.118867Z","shell.execute_reply.started":"2024-11-22T07:40:37.852771Z","shell.execute_reply":"2024-11-22T07:40:45.118020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import necessary libraries\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\nimport optuna\n\n# Define the Optuna objective function\ndef objective(trial):\n    # Suggest hyperparameters for tuning\n    params = {\n    'objective': 'regression',  # Regression task\n    'metric': 'rmse',  # Evaluation metric (can also use 'mae' or 'mape')\n    'boosting_type': 'gbdt',  # Gradient Boosting Decision Tree\n\n    # Tree-related parameters\n    'num_leaves': trial.suggest_int('num_leaves', 20, 100),  # Max leaves per tree\n    'max_depth': trial.suggest_int('max_depth', -1, 20),  # Maximum tree depth (-1 means no limit)\n    'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 10, 100),  # Min samples per leaf\n    'min_sum_hessian_in_leaf': trial.suggest_loguniform('min_sum_hessian_in_leaf', 1e-3, 10),  # Minimum sum of instance weight (hessian) in a leaf\n\n    # Feature sampling\n    'feature_fraction': trial.suggest_uniform('feature_fraction', 0.6, 1.0),  # Fraction of features used per iteration\n    'feature_fraction_bynode': trial.suggest_uniform('feature_fraction_bynode', 0.6, 1.0),  # Fraction of features used per split\n\n    # Bagging (Row sampling)\n    'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.6, 1.0),  # Fraction of data used per iteration\n    'bagging_freq': trial.suggest_int('bagging_freq', 1, 10),  # Frequency for bagging\n    'bagging_seed': 42,  # Seed for bagging\n\n    # Regularization\n    'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),  # L1 regularization term\n    'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),  # L2 regularization term\n    'min_gain_to_split': trial.suggest_loguniform('min_gain_to_split', 1e-8, 10.0),  # Minimum gain to perform split\n\n    # Learning rate and boosting\n    'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.1),  # Step size\n    'num_boost_round': trial.suggest_int('num_boost_round', 100, 1000),  # Number of boosting iterations\n\n    # Early stopping\n    'early_stopping_round': trial.suggest_int('early_stopping_round', 10, 50),  # Stop if no improvement\n    \n    # Optimization and parallelism\n    'max_bin': trial.suggest_int('max_bin', 100, 500),  # Max bins for discretizing continuous features\n    'n_estimators': trial.suggest_int('n_estimators', 100, 1000),  # Number of trees\n    'boost_from_average': trial.suggest_categorical('boost_from_average', [True, False]),  # Adjust initial score\n    'verbosity': -1,  # Suppresses logs\n\n    }\n\n\n    # Train LightGBM model with suggested hyperparameters\n    lgb_train = lgb.Dataset(x_train_prep, y_train)\n    lgb_valid = lgb.Dataset(x_test_prep, y_test, reference=lgb_train)\n    \n    model = lgb.train(params, lgb_train, valid_sets=[lgb_valid], \n                  callbacks=[lgb.early_stopping(10), lgb.log_evaluation(-1)])  # -1 suppresses logging\n\n\n    \n    # Make predictions and evaluate\n    y_pred = model.predict(x_test_prep)\n    rmse = mean_squared_error(y_test, y_pred, squared=False)  # Calculate RMSE\n    return rmse\n\n# Optimize hyperparameters with Optuna\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)  # Number of trials (can be adjusted)\n\n# Display the best hyperparameters\nprint(\"Best trial:\", study.best_trial.params)\n\n# Train the final model using the best hyperparameters\nbest_params = study.best_trial.params\nfinal_model = lgb.train(best_params, lgb.Dataset(x_train_prep, y_train), num_boost_round=100)\n\n# Evaluate the final model\ny_pred_final = final_model.predict(x_test_prep)\nrmse_final = mean_squared_error(y_test, y_pred_final, squared=False)\nprint(f'Final RMSE: {rmse_final:.4f}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T08:18:42.050035Z","iopub.execute_input":"2024-11-22T08:18:42.050466Z","iopub.status.idle":"2024-11-22T08:38:18.527512Z","shell.execute_reply.started":"2024-11-22T08:18:42.050433Z","shell.execute_reply":"2024-11-22T08:38:18.526120Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import  r2_score\n\nr2 = r2_score(y_test,y_pred_final)\n\nprint(f\"R2 Score (rounded predictions): {r2:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T08:38:19.201874Z","iopub.execute_input":"2024-11-22T08:38:19.202778Z","iopub.status.idle":"2024-11-22T08:38:19.209281Z","shell.execute_reply.started":"2024-11-22T08:38:19.202730Z","shell.execute_reply":"2024-11-22T08:38:19.208544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preprocessed = preprocessor.fit_transform(test_preprocessed)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T08:38:24.107280Z","iopub.execute_input":"2024-11-22T08:38:24.107673Z","iopub.status.idle":"2024-11-22T08:38:24.930855Z","shell.execute_reply.started":"2024-11-22T08:38:24.107644Z","shell.execute_reply":"2024-11-22T08:38:24.929945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bg = final_model.predict(test_preprocessed)\nprint(len(bg))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T08:38:31.131792Z","iopub.execute_input":"2024-11-22T08:38:31.132680Z","iopub.status.idle":"2024-11-22T08:38:31.157128Z","shell.execute_reply.started":"2024-11-22T08:38:31.132640Z","shell.execute_reply":"2024-11-22T08:38:31.156218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.DataFrame()\nt = pd.read_csv('/kaggle/input/brist1d/test.csv')\ndf['id'] = t['id']\ndf['bg+1:00'] = bg\ndf.to_csv('submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T08:53:51.668442Z","iopub.execute_input":"2024-11-22T08:53:51.668830Z","iopub.status.idle":"2024-11-22T08:53:51.828058Z","shell.execute_reply.started":"2024-11-22T08:53:51.668799Z","shell.execute_reply":"2024-11-22T08:53:51.827076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T08:44:48.410684Z","iopub.execute_input":"2024-11-22T08:44:48.411092Z","iopub.status.idle":"2024-11-22T08:44:48.421624Z","shell.execute_reply.started":"2024-11-22T08:44:48.411060Z","shell.execute_reply":"2024-11-22T08:44:48.420967Z"}},"outputs":[],"execution_count":null}]}