{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:23.639630Z","iopub.execute_input":"2024-11-22T23:47:23.640223Z","iopub.status.idle":"2024-11-22T23:47:23.649191Z","shell.execute_reply.started":"2024-11-22T23:47:23.640167Z","shell.execute_reply":"2024-11-22T23:47:23.646962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/brist1d/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/brist1d/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:23.674709Z","iopub.execute_input":"2024-11-22T23:47:23.675392Z","iopub.status.idle":"2024-11-22T23:47:33.545726Z","shell.execute_reply.started":"2024-11-22T23:47:23.675335Z","shell.execute_reply":"2024-11-22T23:47:33.544492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Shape of train ',train.shape)\nprint(f'shape of test {test.shape} ')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:33.547690Z","iopub.execute_input":"2024-11-22T23:47:33.548051Z","iopub.status.idle":"2024-11-22T23:47:33.554080Z","shell.execute_reply.started":"2024-11-22T23:47:33.548016Z","shell.execute_reply":"2024-11-22T23:47:33.552857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.sample(4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:33.555443Z","iopub.execute_input":"2024-11-22T23:47:33.555800Z","iopub.status.idle":"2024-11-22T23:47:33.594895Z","shell.execute_reply.started":"2024-11-22T23:47:33.555762Z","shell.execute_reply":"2024-11-22T23:47:33.593668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.sample(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:33.597317Z","iopub.execute_input":"2024-11-22T23:47:33.597704Z","iopub.status.idle":"2024-11-22T23:47:33.620781Z","shell.execute_reply.started":"2024-11-22T23:47:33.597667Z","shell.execute_reply":"2024-11-22T23:47:33.619301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"uniq_cols=[]\nfor i in train.columns:\n    col = i.split('-')[0]\n    uniq_cols.append(col)\nuniq_cols = np.unique(uniq_cols)\nprint(uniq_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:33.622253Z","iopub.execute_input":"2024-11-22T23:47:33.622725Z","iopub.status.idle":"2024-11-22T23:47:33.639108Z","shell.execute_reply.started":"2024-11-22T23:47:33.622689Z","shell.execute_reply":"2024-11-22T23:47:33.637981Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### On each unique col I am making a separate df to check each features relation with target then startegy for missing values will be developed","metadata":{}},{"cell_type":"code","source":"uniq_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:33.640436Z","iopub.execute_input":"2024-11-22T23:47:33.640766Z","iopub.status.idle":"2024-11-22T23:47:33.654810Z","shell.execute_reply.started":"2024-11-22T23:47:33.640732Z","shell.execute_reply":"2024-11-22T23:47:33.653673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## this function will make a separate df for each col see the result to understand\ndef df_of_col (train,col):\n    df = pd.DataFrame()\n    for i in train.columns:\n        if (i.split('-')[0] ==col) & (i  not in ['hr','bg+1:00','id','p_num']):\n            df[i] = train[i]\n    return df\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:33.656106Z","iopub.execute_input":"2024-11-22T23:47:33.656412Z","iopub.status.idle":"2024-11-22T23:47:33.668188Z","shell.execute_reply.started":"2024-11-22T23:47:33.656380Z","shell.execute_reply":"2024-11-22T23:47:33.666873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"activity_df = df_of_col(train,'activity')\nactivity_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:33.669917Z","iopub.execute_input":"2024-11-22T23:47:33.670381Z","iopub.status.idle":"2024-11-22T23:47:33.795666Z","shell.execute_reply.started":"2024-11-22T23:47:33.670330Z","shell.execute_reply":"2024-11-22T23:47:33.794602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"activity = df_of_col(train,uniq_cols[0])\nbg =  df_of_col(train,uniq_cols[1])\ncals =  df_of_col(train,uniq_cols[3])\ncarbs =  df_of_col(train,uniq_cols[4])\ninsulin =  df_of_col(train,'insulin')\nsteps =  df_of_col(train,uniq_cols[-2])\ntime =  df_of_col(train,uniq_cols[-1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:33.797084Z","iopub.execute_input":"2024-11-22T23:47:33.797505Z","iopub.status.idle":"2024-11-22T23:47:34.199819Z","shell.execute_reply.started":"2024-11-22T23:47:33.797464Z","shell.execute_reply":"2024-11-22T23:47:34.198685Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" ## target feature's distribution","metadata":{}},{"cell_type":"code","source":"target = train['bg+1:00']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:34.203503Z","iopub.execute_input":"2024-11-22T23:47:34.203860Z","iopub.status.idle":"2024-11-22T23:47:34.211915Z","shell.execute_reply.started":"2024-11-22T23:47:34.203826Z","shell.execute_reply":"2024-11-22T23:47:34.210302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.distplot(target)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:34.213759Z","iopub.execute_input":"2024-11-22T23:47:34.214135Z","iopub.status.idle":"2024-11-22T23:47:35.321182Z","shell.execute_reply.started":"2024-11-22T23:47:34.214096Z","shell.execute_reply":"2024-11-22T23:47:35.320004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'skewness in bg+1:00 is {target.skew()}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:35.323064Z","iopub.execute_input":"2024-11-22T23:47:35.323517Z","iopub.status.idle":"2024-11-22T23:47:35.332359Z","shell.execute_reply.started":"2024-11-22T23:47:35.323480Z","shell.execute_reply":"2024-11-22T23:47:35.331098Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## activity features and target","metadata":{}},{"cell_type":"code","source":"activity_missing_percentage = activity.isnull().mean() * 100\nprint(activity_missing_percentage)\nprint(\"------------------------------------------------------------------------------\")\nprint(f'mean percentage for missing values is {np.mean(activity_missing_percentage)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:35.334040Z","iopub.execute_input":"2024-11-22T23:47:35.334567Z","iopub.status.idle":"2024-11-22T23:47:35.702788Z","shell.execute_reply.started":"2024-11-22T23:47:35.334505Z","shell.execute_reply":"2024-11-22T23:47:35.701536Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'on average this much values are missing for activity columns : {np.mean(activity.isnull().sum())}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:35.704908Z","iopub.execute_input":"2024-11-22T23:47:35.705374Z","iopub.status.idle":"2024-11-22T23:47:36.078260Z","shell.execute_reply.started":"2024-11-22T23:47:35.705321Z","shell.execute_reply":"2024-11-22T23:47:36.076905Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### so its better to drop alll columns of activity from training","metadata":{}},{"cell_type":"code","source":"train_preprocessed = train.drop(columns=activity.columns)\ntest_preprocessed = test.drop(columns=activity.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:36.079702Z","iopub.execute_input":"2024-11-22T23:47:36.080153Z","iopub.status.idle":"2024-11-22T23:47:36.287733Z","shell.execute_reply.started":"2024-11-22T23:47:36.080100Z","shell.execute_reply":"2024-11-22T23:47:36.286537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tabulate import tabulate\nfeatures = pd.Series(['bg','cals','carbs','insulin','steps'])\nfe_list = [bg,cals,carbs,insulin,steps]\nmean_skewness = [np.mean(i.skew()) for i in fe_list]\ntable_data = list(zip(features,mean_skewness))\nheaders = ['Feature','Avg_skewness']\ntable = tabulate(table_data,headers,tablefmt='pretty')\nprint(table)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:36.288750Z","iopub.execute_input":"2024-11-22T23:47:36.289066Z","iopub.status.idle":"2024-11-22T23:47:36.961162Z","shell.execute_reply.started":"2024-11-22T23:47:36.289034Z","shell.execute_reply":"2024-11-22T23:47:36.959899Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Null Value Analaysis","metadata":{}},{"cell_type":"code","source":"from tabulate import tabulate\nfeatures = pd.Series(['bg','cals','carbs','insulin','steps'])\nfe_list = [bg,cals,carbs,insulin,steps]\nmean_nulls = [np.mean(i.isnull().sum()) for i in fe_list]\nmean_percents =[np.mean(i.isnull().mean()*100) for i in fe_list]\ntable_data = list(zip(features,mean_nulls,mean_percents))\nheaders = ['Feature','Avg_nulls','Avg_null_percents']\ntable = tabulate(table_data,headers,tablefmt='pretty')\nprint(table)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:36.962646Z","iopub.execute_input":"2024-11-22T23:47:36.963028Z","iopub.status.idle":"2024-11-22T23:47:37.216379Z","shell.execute_reply.started":"2024-11-22T23:47:36.962945Z","shell.execute_reply":"2024-11-22T23:47:37.215012Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### as carbs also have almost all missing data so dropping it","metadata":{}},{"cell_type":"code","source":"train_preprocessed = train_preprocessed.drop(columns=carbs.columns)\ntest_preprocessed = test_preprocessed.drop(columns=carbs.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:37.217945Z","iopub.execute_input":"2024-11-22T23:47:37.218311Z","iopub.status.idle":"2024-11-22T23:47:37.401997Z","shell.execute_reply.started":"2024-11-22T23:47:37.218274Z","shell.execute_reply":"2024-11-22T23:47:37.400779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_preprocessed.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:37.403481Z","iopub.execute_input":"2024-11-22T23:47:37.403905Z","iopub.status.idle":"2024-11-22T23:47:37.412572Z","shell.execute_reply.started":"2024-11-22T23:47:37.403817Z","shell.execute_reply":"2024-11-22T23:47:37.411016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"### you will see in further correlation analysis that steps have negligible relation with target and 53%  missing values on average so dropping all steps\ntrain_preprocessed = train_preprocessed.drop(columns=steps.columns)\ntest_preprocessed = test_preprocessed.drop(columns=steps.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:37.414140Z","iopub.execute_input":"2024-11-22T23:47:37.414661Z","iopub.status.idle":"2024-11-22T23:47:37.570068Z","shell.execute_reply.started":"2024-11-22T23:47:37.414608Z","shell.execute_reply":"2024-11-22T23:47:37.568823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cals_missing = train_preprocessed[cals.columns].isnull().any(axis=1)\nbg_missing = train_preprocessed[bg.columns].isnull().any(axis=1)  # Replace bg_columns with actual bg column names\ninsulin_missing = train_preprocessed[insulin.columns].isnull().any(axis=1)  # Replace insulin_columns with actual insulin column names\n\n# Find rows where all three categories are missing\nall_missing = cals_missing & bg_missing & insulin_missing\n\nprint(\"Rows with all cals, bg, and insulin missing:\", all_missing.sum())\nprint(\"Rows with only cals missing:\", cals_missing.sum() - all_missing.sum())\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:37.571631Z","iopub.execute_input":"2024-11-22T23:47:37.572026Z","iopub.status.idle":"2024-11-22T23:47:37.692531Z","shell.execute_reply.started":"2024-11-22T23:47:37.571990Z","shell.execute_reply":"2024-11-22T23:47:37.691383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preprocessed.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:37.693989Z","iopub.execute_input":"2024-11-22T23:47:37.694449Z","iopub.status.idle":"2024-11-22T23:47:37.702189Z","shell.execute_reply.started":"2024-11-22T23:47:37.694379Z","shell.execute_reply":"2024-11-22T23:47:37.701064Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### remaining null values will be imputed","metadata":{}},{"cell_type":"markdown","source":"## Correlation analysis","metadata":{}},{"cell_type":"code","source":"features = pd.Series(['bg', 'cals', 'insulin'])\nfe_list = ['bg', 'cals', 'insulin']  # These are column prefixes\n\nmean_skewness = [\n    np.mean(train_preprocessed.filter(like=col_prefix).skew()) for col_prefix in fe_list\n]\n\n# Create table with results\ntable_data = list(zip(features, mean_skewness))\nheaders = ['Feature', 'Avg_skewness']\ntable = tabulate(table_data, headers, tablefmt='pretty')\n\nprint(table)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:37.704014Z","iopub.execute_input":"2024-11-22T23:47:37.704909Z","iopub.status.idle":"2024-11-22T23:47:38.138194Z","shell.execute_reply.started":"2024-11-22T23:47:37.704851Z","shell.execute_reply":"2024-11-22T23:47:38.137018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nfig, ax = plt.subplots(3, 1, figsize=(12, 15))  # Adjusted for 3 rows and 1 column\n\n# Plot each scatter plot in a separate subplot\nsns.scatterplot(x='bg-0:20', y='bg+1:00', data=train_preprocessed, ax=ax[0])\nsns.scatterplot(x='insulin-0:20', y='bg+1:00', data=train_preprocessed, ax=ax[1])\nsns.scatterplot(x='cals-1:50', y='bg+1:00', data=train_preprocessed, ax=ax[2])\n\n# Add titles and adjust layout\nax[0].set_title('bg-0:20 vs bg+1:00')\nax[1].set_title('insulin-0:20 vs bg+1:00')\nax[2].set_title('cals-1:50 vs bg+1:00')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:38.139759Z","iopub.execute_input":"2024-11-22T23:47:38.140661Z","iopub.status.idle":"2024-11-22T23:47:39.808451Z","shell.execute_reply.started":"2024-11-22T23:47:38.140608Z","shell.execute_reply":"2024-11-22T23:47:39.807268Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Why We Are Using Spearman Correlation\r\n\r\nWe chose Spearman correlation to assess the relationship between **bg**, **cals**, and **insulin** features with the target due to two key factors:\r\n\r\n1. **Skewness in Data**:\r\n   - The target variable has a skewness of **1.07**, and the average skewness in the insulin features is **1.06**, indicating a non-normal distribution across these variables. Both the **bg** and **cals** feature sets also exhibit similar skewness levels, reinforcing the choice to use Spearman correlation. Unlike Pearson correlation, which is sensitive to skewed data, Spearman correlation is robust against non-normal distributions, making it a better fit for our data.\r\n\r\n2. **Non-linear Relationship**:\r\n   - The scatterplots between key features like **bg['bg-0:20']**, **cals['cals-1:50']**, **insulin['insulin-0:20']**, and the target demonstrate non-linear, monotonic relationships rather than clear linear patterns. Spearman correlation captures monotonic relationships, even if they are not strictly linear, making it more suitable for assessing the associations within our data.\r\n\r\nIn summary, Spearman correlation offers a robust approach that accounts for both the skewness and the monotonic relationships observed between the features and the target.\r\ncontext.","metadata":{}},{"cell_type":"code","source":"bg_corr_spearman = bg.corrwith(target, method='spearman').sort_values(ascending=False)\ncals_corr_spearman = cals.corrwith(target, method='spearman').sort_values(ascending=False)\ninsulin_corr_spearman = insulin.corrwith(target, method='spearman').sort_values(ascending=False)\n\n# Print Spearman correlations for bg\nprint(\"Spearman Correlations for bg:\")\nprint(bg_corr_spearman)\nprint(f'min bg_corr_spearman is {bg_corr_spearman.min()}')\nprint(f'max bg_corr_spearman is {bg_corr_spearman.max()}')\nprint(f'mean bg_corr_spearman is {bg_corr_spearman.mean()}\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:39.810022Z","iopub.execute_input":"2024-11-22T23:47:39.810452Z","iopub.status.idle":"2024-11-22T23:47:46.509790Z","shell.execute_reply.started":"2024-11-22T23:47:39.810390Z","shell.execute_reply":"2024-11-22T23:47:46.508636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Spearman Correlations:\")\nprint(cals_corr_spearman)\nprint(f'min cals_corr_spearman is {cals_corr_spearman.min()}')\nprint(f'max bg_corr_spearman is {cals_corr_spearman.max()}')\nprint(f' mean calsinsulin_corr_spearman is {cals_corr_spearman.mean()} ')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:46.511112Z","iopub.execute_input":"2024-11-22T23:47:46.511475Z","iopub.status.idle":"2024-11-22T23:47:46.520262Z","shell.execute_reply.started":"2024-11-22T23:47:46.511438Z","shell.execute_reply":"2024-11-22T23:47:46.518876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Spearman Correlations for insulin:\")\nprint(insulin_corr_spearman)\nprint(f'min insulin_corr_spearman is {insulin_corr_spearman.min()}')\nprint(f'max insulin_corr_spearman is {insulin_corr_spearman.max()}')\nprint(f'mean insulin_corr_spearman is {insulin_corr_spearman.mean()}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:46.521640Z","iopub.execute_input":"2024-11-22T23:47:46.522042Z","iopub.status.idle":"2024-11-22T23:47:46.537900Z","shell.execute_reply.started":"2024-11-22T23:47:46.522009Z","shell.execute_reply":"2024-11-22T23:47:46.536531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cals_insulin_corr = train_preprocessed[cals.columns].corrwith(train_preprocessed[insulin.columns].mean(axis=1), method='spearman').sort_values(ascending=False)\nprint(\"Spearman Correlations between cals and insulin:\")\nprint(cals_insulin_corr)\nprint(f\"Min correlation between cals and insulin: {cals_insulin_corr.min()}\")\nprint(f\"Max correlation between cals and insulin: {cals_insulin_corr.max()}\")\nprint(f\"Mean correlation between cals and insulin: {cals_insulin_corr.mean()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:46.542769Z","iopub.execute_input":"2024-11-22T23:47:46.543214Z","iopub.status.idle":"2024-11-22T23:47:49.205774Z","shell.execute_reply.started":"2024-11-22T23:47:46.543177Z","shell.execute_reply":"2024-11-22T23:47:49.204681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cals_bg_corr = train_preprocessed[cals.columns].corrwith(train_preprocessed[bg.columns].mean(axis=1), method='spearman').sort_values(ascending=False)\nprint(\"\\nSpearman Correlations between cals and bg:\")\nprint(cals_bg_corr)\nprint(f\"Min correlation between cals and bg: {cals_bg_corr.min()}\")\nprint(f\"Max correlation between cals and bg: {cals_bg_corr.max()}\")\nprint(f\"Mean correlation between cals and bg: {cals_bg_corr.mean()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:49.206893Z","iopub.execute_input":"2024-11-22T23:47:49.207176Z","iopub.status.idle":"2024-11-22T23:47:51.972898Z","shell.execute_reply.started":"2024-11-22T23:47:49.207147Z","shell.execute_reply":"2024-11-22T23:47:51.971204Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Rationale for Dropping `cals` Features\r\n\r\nThe `cals` features were dropped due to their minimal correlation with the target variable, `bg`. The Spearman correlation between `cals` and `bg` remained consistently low (mean ≈ -0.043), indicating negligible predictive value. Retaining low-impact features can introduce noise, increase model complexity, and affect training efficiency. By removing `cals`, we focus on the most relevant features, streamlining the model and potentially improving its performance without sacrificing predictive accuracy.\r\n","metadata":{}},{"cell_type":"code","source":"train_preprocessed = train_preprocessed.drop(columns=cals.columns)\ntest_preprocessed = test_preprocessed.drop(columns=cals.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:51.974707Z","iopub.execute_input":"2024-11-22T23:47:51.975221Z","iopub.status.idle":"2024-11-22T23:47:52.098610Z","shell.execute_reply.started":"2024-11-22T23:47:51.975170Z","shell.execute_reply":"2024-11-22T23:47:52.096978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"steps_corr_spearman = steps.corrwith(target, method='spearman').sort_values(ascending=False)\nprint(f'min steps_corr_spearman is {steps_corr_spearman.min()}')\nprint(f'maxsteps_corr_spearman is {steps_corr_spearman.max()}')\nprint(f' mean calsinsulin_corr_spearman is {steps_corr_spearman.mean()} ')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:52.100712Z","iopub.execute_input":"2024-11-22T23:47:52.101342Z","iopub.status.idle":"2024-11-22T23:47:53.472404Z","shell.execute_reply.started":"2024-11-22T23:47:52.101283Z","shell.execute_reply":"2024-11-22T23:47:53.471146Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### why we dropped steps too","metadata":{}},{"cell_type":"code","source":"np.mean(train_preprocessed[insulin.columns].skew())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:53.473970Z","iopub.execute_input":"2024-11-22T23:47:53.474344Z","iopub.status.idle":"2024-11-22T23:47:53.622313Z","shell.execute_reply.started":"2024-11-22T23:47:53.474307Z","shell.execute_reply":"2024-11-22T23:47:53.621146Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engineering pipeline","metadata":{}},{"cell_type":"code","source":"train_preprocessed.drop(columns=['id','p_num','time'],inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:47:53.623796Z","iopub.execute_input":"2024-11-22T23:47:53.624251Z","iopub.status.idle":"2024-11-22T23:47:53.733450Z","shell.execute_reply.started":"2024-11-22T23:47:53.624201Z","shell.execute_reply":"2024-11-22T23:47:53.732098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, PowerTransformer\nfrom sklearn.pipeline import Pipeline\nx = train_preprocessed.drop(columns=['bg+1:00'])\ny = train_preprocessed['bg+1:00']\n\n# Assuming X_train is your DataFrame\n# Extract only the numeric columns from the DataFrame\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)\n\n# Select numerical and categorical features\nnumeric_features = x.select_dtypes(include=['int64', 'float64']).columns.tolist()\ncat_features = x.select_dtypes(include='object').columns.tolist()  # Use 'x' instead of 'x_train'\n\n# Define the numerical pipeline (impute missing values + apply power transformation)\nnumerical_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy='median')),            \n    ('power', PowerTransformer(method='yeo-johnson')) \n])\n\n# Create the ColumnTransformer to apply the numerical pipeline to the numeric columns\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numerical_pipeline, numeric_features),\n    ]\n)\n\n# Fit the preprocessor on the training data and then transform the training and test data\nx_train_prep = preprocessor.fit_transform(x_train)  # Fit and transform on training data\nx_test_prep = preprocessor.transform(x_test)       # Only transform on test data\n\n# You can check the transformed data\nprint(x_train_prep)\nprint(x_test_prep)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:48:05.118478Z","iopub.execute_input":"2024-11-22T23:48:05.120116Z","iopub.status.idle":"2024-11-22T23:48:46.074622Z","shell.execute_reply.started":"2024-11-22T23:48:05.120041Z","shell.execute_reply":"2024-11-22T23:48:46.073494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preprocessed = preprocessor.fit_transform(test_preprocessed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:48:46.076992Z","iopub.execute_input":"2024-11-22T23:48:46.077486Z","iopub.status.idle":"2024-11-22T23:48:47.364491Z","shell.execute_reply.started":"2024-11-22T23:48:46.077413Z","shell.execute_reply":"2024-11-22T23:48:47.362955Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature selection","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import cross_val_score\n\n# Define the objective function\ndef objective(trial):\n    # Suggest values for the hyperparameters\n    n_estimators = trial.suggest_int('n_estimators', 50, 200)  # Number of trees in the forest\n    max_depth = trial.suggest_int('max_depth', 5, 30)            # Maximum depth of the trees\n    min_samples_split = trial.suggest_int('min_samples_split', 2, 10)  # Minimum samples required to split an internal node\n    min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 10)  # Minimum samples required to be at a leaf node\n    max_features = trial.suggest_categorical('max_features', ['auto', 'sqrt', 'log2'])  # The number of features to consider when looking for the best split\n\n    # Create the RandomForestRegressor with suggested hyperparameters\n    model = RandomForestRegressor(\n        n_estimators=n_estimators,\n        max_depth=max_depth,\n        min_samples_split=min_samples_split,\n        min_samples_leaf=min_samples_leaf,\n        max_features=max_features,\n        random_state=42\n    )\n\n    # Perform 3-fold cross-validation and calculate RMSE\n    score = cross_val_score(model, X_train, y_train, cv=3, scoring='neg_root_mean_squared_error').mean()\n\n    return score  # Return negative RMSE since cross_val_score returns negative values\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:48:47.366223Z","iopub.execute_input":"2024-11-22T23:48:47.366712Z","iopub.status.idle":"2024-11-22T23:48:47.375158Z","shell.execute_reply.started":"2024-11-22T23:48:47.366661Z","shell.execute_reply":"2024-11-22T23:48:47.374027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a study object and optimize the objective function\n##import optuna\n##study = optuna.create_study(direction='minimize', sampler=optuna.samplers.TPESampler())  # We aim to maximize accuracy\n##study.optimize(objective, n_trials=50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-22T23:48:47.377976Z","iopub.execute_input":"2024-11-22T23:48:47.379118Z","iopub.status.idle":"2024-11-22T23:48:47.789990Z","shell.execute_reply.started":"2024-11-22T23:48:47.379074Z","shell.execute_reply":"2024-11-22T23:48:47.788242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}