{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-26T08:31:01.628616Z","iopub.execute_input":"2024-10-26T08:31:01.629644Z","iopub.status.idle":"2024-10-26T08:31:02.108209Z","shell.execute_reply.started":"2024-10-26T08:31:01.629575Z","shell.execute_reply":"2024-10-26T08:31:02.106927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References and links\n### Reference notebboks\n1. https://www.kaggle.com/code/docxian/blood-glucose-prediction-explorations#Model","metadata":{}},{"cell_type":"markdown","source":"# Loading Training Data and general EDA ","metadata":{}},{"cell_type":"code","source":"#load the data\ntrain_csv = pd.read_csv(\"/kaggle/input/brist1d/train.csv\")\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T08:31:05.197054Z","iopub.execute_input":"2024-10-26T08:31:05.197663Z","iopub.status.idle":"2024-10-26T08:31:20.990154Z","shell.execute_reply.started":"2024-10-26T08:31:05.197617Z","shell.execute_reply":"2024-10-26T08:31:20.988987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# see the columns\ntrain_csv.columns","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:39.718103Z","iopub.execute_input":"2024-10-12T12:52:39.718528Z","iopub.status.idle":"2024-10-12T12:52:39.726263Z","shell.execute_reply.started":"2024-10-12T12:52:39.718481Z","shell.execute_reply":"2024-10-12T12:52:39.725079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  training data description\ntrain_csv.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:39.728092Z","iopub.execute_input":"2024-10-12T12:52:39.728672Z","iopub.status.idle":"2024-10-12T12:52:43.928472Z","shell.execute_reply.started":"2024-10-12T12:52:39.728616Z","shell.execute_reply":"2024-10-12T12:52:43.927341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# column wise getting NaN values\ntrain_csv.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:43.931957Z","iopub.execute_input":"2024-10-12T12:52:43.932356Z","iopub.status.idle":"2024-10-12T12:52:44.430719Z","shell.execute_reply.started":"2024-10-12T12:52:43.932318Z","shell.execute_reply":"2024-10-12T12:52:44.429445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting the percentage of missing values\nmissing_values = train_csv.isnull().sum()\n\n\ntotal_rows = train_csv.shape[0]\nmissing_percentage = (missing_values / total_rows) * 100\n\n\nresults_df = pd.DataFrame({'Column': missing_values.index,\n                          'Missing Values': missing_values.values,\n                          'Missing Percentage (%)': missing_percentage.values})\n\n\nresults_df = results_df.sort_values(by='Missing Percentage (%)', ascending=False)\nresults_df.to_csv(\"/kaggle/working/percentage_of_missing_values.csv\")\nprint(results_df)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:44.432905Z","iopub.execute_input":"2024-10-12T12:52:44.433268Z","iopub.status.idle":"2024-10-12T12:52:44.946363Z","shell.execute_reply.started":"2024-10-12T12:52:44.433232Z","shell.execute_reply":"2024-10-12T12:52:44.945053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the structure of the data\ntrain_csv.info(show_counts=True, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:44.948214Z","iopub.execute_input":"2024-10-12T12:52:44.948695Z","iopub.status.idle":"2024-10-12T12:52:45.512614Z","shell.execute_reply.started":"2024-10-12T12:52:44.948627Z","shell.execute_reply":"2024-10-12T12:52:45.511386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting the time range\ntrain_csv.time.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:45.513920Z","iopub.execute_input":"2024-10-12T12:52:45.514276Z","iopub.status.idle":"2024-10-12T12:52:45.548491Z","shell.execute_reply.started":"2024-10-12T12:52:45.514241Z","shell.execute_reply":"2024-10-12T12:52:45.547353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert time to datetime dtype of pands\ntrain_csv.time = pd.to_datetime(train_csv.time, format='%H:%M:%S')\ntrain_csv.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:45.549835Z","iopub.execute_input":"2024-10-12T12:52:45.550250Z","iopub.status.idle":"2024-10-12T12:52:45.605419Z","shell.execute_reply.started":"2024-10-12T12:52:45.550214Z","shell.execute_reply":"2024-10-12T12:52:45.604370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.columns[train_csv.columns.str.startswith('cals-')]","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:45.606854Z","iopub.execute_input":"2024-10-12T12:52:45.607240Z","iopub.status.idle":"2024-10-12T12:52:45.616029Z","shell.execute_reply.started":"2024-10-12T12:52:45.607201Z","shell.execute_reply":"2024-10-12T12:52:45.614828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.time.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:45.617429Z","iopub.execute_input":"2024-10-12T12:52:45.617784Z","iopub.status.idle":"2024-10-12T12:52:45.636385Z","shell.execute_reply.started":"2024-10-12T12:52:45.617748Z","shell.execute_reply":"2024-10-12T12:52:45.635008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.time.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:45.638265Z","iopub.execute_input":"2024-10-12T12:52:45.638762Z","iopub.status.idle":"2024-10-12T12:52:45.651861Z","shell.execute_reply.started":"2024-10-12T12:52:45.638710Z","shell.execute_reply":"2024-10-12T12:52:45.650541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# patient ids - training\ncount_train = train_csv.p_num.value_counts().sort_index()\ncount_train","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:45.653433Z","iopub.execute_input":"2024-10-12T12:52:45.653905Z","iopub.status.idle":"2024-10-12T12:52:45.679978Z","shell.execute_reply.started":"2024-10-12T12:52:45.653855Z","shell.execute_reply":"2024-10-12T12:52:45.678483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:45.685102Z","iopub.execute_input":"2024-10-12T12:52:45.686207Z","iopub.status.idle":"2024-10-12T12:52:45.696364Z","shell.execute_reply.started":"2024-10-12T12:52:45.686150Z","shell.execute_reply":"2024-10-12T12:52:45.695089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting Data Distributions","metadata":{}},{"cell_type":"markdown","source":"## Plotting Target data distribution","metadata":{}},{"cell_type":"code","source":"# get the target distribution\n# plot target distribution\naverage_value = train_csv['bg+1:00'].mean()\nimport matplotlib.pyplot as plt\nplt.figure(figsize=(12,4))\ntrain_csv['bg+1:00'].plot(kind='hist', bins=50)\nplt.title('bg+1:00')\n# Add a vertical line for the average value\nplt.axvline(average_value, color='red', linestyle='dashed', linewidth=2)\n# Add annotation for the average value\nplt.text(average_value, plt.gca().get_ylim()[1]*0.9, f'Avg: {average_value:.2f}', color='red', fontsize=12)\nplt.grid()\nplt.show()\n# basic stats\nprint(train_csv['bg+1:00'].describe())","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:45.697698Z","iopub.execute_input":"2024-10-12T12:52:45.698090Z","iopub.status.idle":"2024-10-12T12:52:46.028954Z","shell.execute_reply.started":"2024-10-12T12:52:45.698026Z","shell.execute_reply":"2024-10-12T12:52:46.027460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plotting Time Distributiom","metadata":{}},{"cell_type":"code","source":"# extract hour\n\n\"\"\"\nPurpose : The result is a sorted count of how many times each unique hour value appears in the train_csv DataFrame.\n\"\"\"\ntrain_csv['hour'] = train_csv.time.apply(lambda x : x.hour)\n# and show plot\nplt.figure(figsize=(12,4))\ntrain_csv.hour.value_counts().sort_index().plot(kind='bar', color='red')\nplt.title('Distribution of hour - Training')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:46.030285Z","iopub.execute_input":"2024-10-12T12:52:46.030668Z","iopub.status.idle":"2024-10-12T12:52:46.857567Z","shell.execute_reply.started":"2024-10-12T12:52:46.030605Z","shell.execute_reply":"2024-10-12T12:52:46.856257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plotting target distribution w.r.t time (Violin Plot)","metadata":{}},{"cell_type":"code","source":"# plotting target vs hour\n# plot target distribution by hour\nimport seaborn as sns\nplt.figure(figsize=(14,6))\nsns.violinplot(data=train_csv, x='hour', y='bg+1:00')\nplt.title('Target by hour')\nplt.grid()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:46.859148Z","iopub.execute_input":"2024-10-12T12:52:46.859609Z","iopub.status.idle":"2024-10-12T12:52:48.101802Z","shell.execute_reply.started":"2024-10-12T12:52:46.859560Z","shell.execute_reply":"2024-10-12T12:52:48.100462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The following observations are made:\n 1. The median blood glucose reduces a bit from hour 0 and then from hour 7 it starts increasing  with a slight dip at 4 PM and then again a slight incrase from 5 PM onwards.\n \n 2. Might happen that during breakfast, lunch and dinner there is increase of blood sugar level.","metadata":{}},{"cell_type":"markdown","source":"### Activities List\n\n1. Indoor\n2. climbing\n3. Run\n4. Strength training\n5. Swim\n6. Bike\n7. Dancing\n8. Stairclimber\n9. Spinning\n10. Walking\n11. HIIT\n12. OutdoorBike\n13. Walk\n14. Aerobic Workout\n15. Tennis\n16. Workout\n17. Hike\n18. Zumba\n19. Sport\n20. Yoga\n21. Swimming\n22. Weights\n23. Running\n\n","metadata":{}},{"cell_type":"markdown","source":"## Plotting target distribution w.r.t Patient","metadata":{}},{"cell_type":"code","source":"# plot target distribution by patient\nplt.figure(figsize=(12,5))\nsns.violinplot(data=train_csv, x='p_num', y='bg+1:00')\nplt.title('Target by patient')\nplt.grid()\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:48.103934Z","iopub.execute_input":"2024-10-12T12:52:48.104380Z","iopub.status.idle":"2024-10-12T12:52:49.399312Z","shell.execute_reply.started":"2024-10-12T12:52:48.104335Z","shell.execute_reply":"2024-10-12T12:52:49.398127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\nsns.violinplot(data=train_csv, x='p_num', y='bg+1:00')\n# Calculate the medians for each category in 'p_num'\nmedians = train_csv.groupby('p_num')['bg+1:00'].median().values\n# Get the positions of each category on the x-axis\npositions = range(len(medians))\n# Overlay the median values using plt.scatter (or plt.plot for connecting lines)\nplt.scatter(positions, medians, color='red', marker='o', label='Median')\nfor i, median in enumerate(medians):\n    plt.text(i, median, f'{median:.2f}', color='black', ha='center', va='bottom')\n# Optionally, add a legend\nplt.legend()\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:49.400799Z","iopub.execute_input":"2024-10-12T12:52:49.401185Z","iopub.status.idle":"2024-10-12T12:52:50.850060Z","shell.execute_reply.started":"2024-10-12T12:52:49.401148Z","shell.execute_reply":"2024-10-12T12:52:50.848830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Following points can be made about the patients\n\n1. po1 : has quite a big range of blood sugar value but mostly concerntrated on the lower end.\n2. p02 : Median blood sugar is a bit higher than p01 but not as widespread as p01\n3. p03 : Median almost similar same as p01, most values are conerntrated a bit below the median, not as widesprad as p01\n4. p04 : Median lower previous 3 patients and most of the time p04 have a lower blood sugar than the median as maximum number of data points are concerntrated a bit lower than the median.\n5. p05 : Quite spread out values with maximum number of data points occuring a bit below the median.\n6. p06 : Has a wide range of blood sugar values and the median is 8.10.\n7. p10 : Has the best blood segar level. The range of BS level is quite compact compared to other patients and from the plot it is evident throughout the day the BS is mostly around the median.\n8. p11 : The worst patient until now. Values are quite spread out and evidently most of the data points occur around the medians.\n9. p12 : Most number od data points are concerntrated around the median.\n\n","metadata":{}},{"cell_type":"markdown","source":"# Time Series Analysis #","metadata":{}},{"cell_type":"code","source":"# Adding a column for miniute\ntrain_csv['time_min'] = train_csv.hour * 60 + train_csv.time.apply(lambda x : x.minute)\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:50.852157Z","iopub.execute_input":"2024-10-12T12:52:50.852648Z","iopub.status.idle":"2024-10-12T12:52:51.574665Z","shell.execute_reply.started":"2024-10-12T12:52:50.852581Z","shell.execute_reply":"2024-10-12T12:52:51.573271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unique hour values in our data\nunique_hour_values = [0, 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23]","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:51.576261Z","iopub.execute_input":"2024-10-12T12:52:51.576766Z","iopub.status.idle":"2024-10-12T12:52:51.583137Z","shell.execute_reply.started":"2024-10-12T12:52:51.576722Z","shell.execute_reply":"2024-10-12T12:52:51.581461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Here I want to explore how every hour the blood sugar value is for the patients.\n1. Explore the distribution of target w.r.t the minutes column. From the minutes column we know the time of the day","metadata":{}},{"cell_type":"code","source":"\"\"\"\nfunction to plot the target vs minutes\n\"\"\"\ndef plot_target_vs_minutes(patient, color):\n    df_sel = train_csv[train_csv.p_num==patient]\n    plt.figure(figsize=(14,4))\n    plt.scatter(df_sel.time_min, df_sel[\"bg+1:00\"], color=color, alpha=0.2, s=5)\n    plt.title(patient)\n    plt.grid()\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:51.584772Z","iopub.execute_input":"2024-10-12T12:52:51.585281Z","iopub.status.idle":"2024-10-12T12:52:51.593554Z","shell.execute_reply.started":"2024-10-12T12:52:51.585212Z","shell.execute_reply":"2024-10-12T12:52:51.592313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient = list(set(train_csv[\"p_num\"]))\nprint(patient)\ncolor = ['blue',\n'green',\n'red',\n'purple',\n'orange',\n'pink',\n'brown',\n'gray',\n'olive']\n\nfor p, c in zip(patient, color):\n    plot_target_vs_minutes(p, c)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:51.595048Z","iopub.execute_input":"2024-10-12T12:52:51.595504Z","iopub.status.idle":"2024-10-12T12:52:55.245988Z","shell.execute_reply.started":"2024-10-12T12:52:51.595458Z","shell.execute_reply":"2024-10-12T12:52:55.244821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now the qustion is why p10 and p11 is so different. What p10 is doing so well that the blood suagr is so controlled\n### Hence I would like to explore a little bit with p10 and p11 to get some more insights","metadata":{}},{"cell_type":"code","source":"\"\"\"\nOne way to get this is the total calories burnt time wise\n\"\"\"\ndef plot_calories_vs_time(patient, color):\n    # get the patient df\n    df_patient = train_csv[train_csv[\"p_num\"] == patient]\n    # get all the columns that are associate with calories\n    calorie_columns = list(df_patient.columns[df_patient.columns.str.startswith('cals-')])\n    df_patient[\"avg_calorie\"] = df_patient[calorie_columns].mean(axis=1)\n    plt.figure(figsize=(14,4))\n    plt.scatter(df_patient.time_min, df_patient[\"avg_calorie\"], color=color, alpha=0.2, s=5)\n    plt.title(patient + \"_avg_calories_burnt\")\n    plt.xlabel(\"Minutes\")\n    plt.ylabel(\"Calories burnt\")\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:55.247644Z","iopub.execute_input":"2024-10-12T12:52:55.248158Z","iopub.status.idle":"2024-10-12T12:52:55.258282Z","shell.execute_reply.started":"2024-10-12T12:52:55.248103Z","shell.execute_reply":"2024-10-12T12:52:55.256858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_calories_vs_time(\"p10\", 'red')","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:55.259782Z","iopub.execute_input":"2024-10-12T12:52:55.260175Z","iopub.status.idle":"2024-10-12T12:52:55.689486Z","shell.execute_reply.started":"2024-10-12T12:52:55.260128Z","shell.execute_reply":"2024-10-12T12:52:55.688140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_calories_vs_time(\"p11\", 'blue')","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:55.690941Z","iopub.execute_input":"2024-10-12T12:52:55.691306Z","iopub.status.idle":"2024-10-12T12:52:56.100265Z","shell.execute_reply.started":"2024-10-12T12:52:55.691269Z","shell.execute_reply":"2024-10-12T12:52:56.098978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Comparison of Calorie burnt for p10 and p11","metadata":{}},{"cell_type":"code","source":"# combining the two plots\ndf_patient1 = train_csv[train_csv[\"p_num\"] == \"p10\"]\ndf_patient2 = train_csv[train_csv[\"p_num\"] == \"p11\"]\n# get all the columns that are associate with calories\ncalorie_columns = list(df_patient1.columns[df_patient1.columns.str.startswith('cals-')])\ndf_patient1[\"avg_calorie\"] = df_patient1[calorie_columns].mean(axis=1)\ndf_patient2[\"avg_calorie\"] = df_patient2[calorie_columns].mean(axis=1)\n\nplt.figure(figsize=(14,4))\nplt.scatter(df_patient1.time_min, df_patient1[\"avg_calorie\"], color=\"red\", alpha=0.2, s=5, label = \"p10\")\nplt.scatter(df_patient2.time_min, df_patient2[\"avg_calorie\"], color=\"blue\", alpha=0.2, s=5, label = \"p11\")\nplt.title(\"avg_calories_burnt\")\nplt.xlabel(\"Minutes\")\nplt.ylabel(\"Calories burnt\")\nplt.legend()\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T12:52:56.102031Z","iopub.execute_input":"2024-10-12T12:52:56.102493Z","iopub.status.idle":"2024-10-12T12:52:57.214062Z","shell.execute_reply.started":"2024-10-12T12:52:56.102444Z","shell.execute_reply":"2024-10-12T12:52:57.212958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hence on an average throughout the day we see that p10 burns a lot of calories compared to p11","metadata":{}},{"cell_type":"markdown","source":"# Further EDA\n### With some initial training I could not achieve a good score. The good score that I achieved was only the first one where I practically did nothing but just replaced null values with zero and that's all. There I got a score of 2.54. But after that when I cam discarding columns with missing values I do not get anything useful. Why is that the case?","metadata":{}},{"cell_type":"markdown","source":"## Plotting the actual distribution of missing values","metadata":{}},{"cell_type":"markdown","source":"### Bg/values with blood glucose columns values","metadata":{}},{"cell_type":"code","source":"# get the missing values column wise and plot them\nbg_columns = train_csv.filter(regex='^bg')\nmissing_values = bg_columns.isnull().sum()\ntotal_rows = bg_columns.shape[0]\nmissing_percentage = (missing_values / total_rows) * 100\n\nresults_df = pd.DataFrame({'Column': missing_values.index,\n                          'Missing Values': missing_values.values,\n                          'Missing Percentage (%)': missing_percentage.values})\n\nresults_df = results_df.sort_values(by='Missing Percentage (%)', ascending=False)\nmean_missing_values_bg = results_df[\"Missing Percentage (%)\"].mean()\nprint(\"Mean missing values : \", mean_missing_values_bg)\nprint(results_df.head())","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:28:01.844301Z","iopub.execute_input":"2024-10-26T09:28:01.845118Z","iopub.status.idle":"2024-10-26T09:28:01.913431Z","shell.execute_reply.started":"2024-10-26T09:28:01.845068Z","shell.execute_reply":"2024-10-26T09:28:01.911899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(10, 6))\nplt.bar(results_df['Column'], results_df['Missing Percentage (%)'], color='skyblue')\nplt.xlabel('Column Name')\nplt.ylabel('Missing Percentage (%)')\nplt.title('Missing Values Percentage by Column')\nplt.xticks(rotation=90, fontsize=5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:28:22.145615Z","iopub.execute_input":"2024-10-26T09:28:22.146055Z","iopub.status.idle":"2024-10-26T09:28:22.980965Z","shell.execute_reply.started":"2024-10-26T09:28:22.146013Z","shell.execute_reply":"2024-10-26T09:28:22.979771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we see from the above graph there are missing values for blood glucose measurements as well. maybe the reason that those were not reported as was mentioned in the dataset","metadata":{}},{"cell_type":"markdown","source":"### Missing Percentage Insulin values","metadata":{}},{"cell_type":"code","source":"# get the missing values column wise and plot them\ninsulin_columns = train_csv.filter(regex='^insulin')\nmissing_values = insulin_columns.isnull().sum()\ntotal_rows = insulin_columns.shape[0]\nmissing_percentage = (insulin_columns.isnull().mean() * 100)\n\nresults_df = pd.DataFrame({'Column': missing_values.index,\n                          'Missing Values': missing_values.values,\n                          'Missing Percentage (%)': missing_percentage.values})\n\nresults_df = results_df.sort_values(by='Missing Percentage (%)', ascending=False)\nmean_missing_percentage_insulin = results_df['Missing Percentage (%)'].mean()\nprint(\"mean_missing_percentage_insulin : \", mean_missing_percentage_insulin)\nprint(results_df.head())\n\nplt.figure(figsize=(10, 6))\nplt.bar(results_df['Column'], results_df['Missing Percentage (%)'], color='mediumseagreen')\nplt.xlabel('Column Name')\nplt.ylabel('Missing Percentage (%)')\nplt.title('Missing Values Percentage by Column')\nplt.xticks(rotation=90, fontsize=5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:31:35.231239Z","iopub.execute_input":"2024-10-26T09:31:35.231714Z","iopub.status.idle":"2024-10-26T09:31:36.222457Z","shell.execute_reply.started":"2024-10-26T09:31:35.231671Z","shell.execute_reply":"2024-10-26T09:31:36.221210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is evident that missing percentage is very uniform for Insulin","metadata":{}},{"cell_type":"markdown","source":"### Missing Percenatge for Carbs","metadata":{}},{"cell_type":"code","source":"# get the missing values column wise and plot them\ncarbs_columns = train_csv.filter(regex='^carbs')\nmissing_values = carbs_columns.isnull().sum()\ntotal_rows = carbs_columns.shape[0]\nmissing_percentage = (carbs_columns.isnull().mean() * 100)\n\nresults_df = pd.DataFrame({'Column': missing_values.index,\n                          'Missing Values': missing_values.values,\n                          'Missing Percentage (%)': missing_percentage.values})\n\nmean_missing_percentage_carbs = results_df['Missing Percentage (%)'].mean()\nprint(\"mean_missing_percentage_carbs : \", mean_missing_percentage_carbs)\nresults_df = results_df.sort_values(by='Missing Percentage (%)', ascending=False)\nprint(results_df.head())\n\nplt.figure(figsize=(10, 6))\nplt.bar(results_df['Column'], results_df['Missing Percentage (%)'], color='mediumseagreen')\nplt.xlabel('Column Name')\nplt.ylabel('Missing Percentage (%)')\nplt.title('Missing Values Percentage by Column')\nplt.xticks(rotation=90, fontsize=5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:32:06.011661Z","iopub.execute_input":"2024-10-26T09:32:06.012270Z","iopub.status.idle":"2024-10-26T09:32:07.294107Z","shell.execute_reply.started":"2024-10-26T09:32:06.012208Z","shell.execute_reply":"2024-10-26T09:32:07.292442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is quite disturbing that almost ","metadata":{}},{"cell_type":"markdown","source":"### Heart Rate Missing values","metadata":{}},{"cell_type":"code","source":"# get the missing values column wise and plot them\nhr_columns = train_csv.filter(regex='^hr')\nmissing_values = hr_columns.isnull().sum()\ntotal_rows = hr_columns.shape[0]\nmissing_percentage = (hr_columns.isnull().mean() * 100)\n\nresults_df = pd.DataFrame({'Column': missing_values.index,\n                          'Missing Values': missing_values.values,\n                          'Missing Percentage (%)': missing_percentage.values})\n\nmean_missing_percentage_heart_rate = results_df['Missing Percentage (%)'].mean()\nprint(\"mean_missing_percentage_heart_rate : \", mean_missing_percentage_heart_rate)\nresults_df = results_df.sort_values(by='Missing Percentage (%)', ascending=False)\nprint(results_df.head())\n\nplt.figure(figsize=(10, 6))\nplt.bar(results_df['Column'], results_df['Missing Percentage (%)'], color='mediumseagreen')\nplt.xlabel('Column Name')\nplt.ylabel('Missing Percentage (%)')\nplt.title('Missing Values Percentage by Column')\nplt.xticks(rotation=90, fontsize=5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:33:36.723737Z","iopub.execute_input":"2024-10-26T09:33:36.724211Z","iopub.status.idle":"2024-10-26T09:33:37.883998Z","shell.execute_reply.started":"2024-10-26T09:33:36.724169Z","shell.execute_reply":"2024-10-26T09:33:37.882790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Total Steps walked missing values","metadata":{}},{"cell_type":"code","source":"# get the missing values column wise and plot them\nsteps_columns = train_csv.filter(regex='^steps')\nmissing_values = steps_columns.isnull().sum()\ntotal_rows = steps_columns.shape[0]\nmissing_percentage = (steps_columns.isnull().mean() * 100)\n\nresults_df = pd.DataFrame({'Column': missing_values.index,\n                          'Missing Values': missing_values.values,\n                          'Missing Percentage (%)': missing_percentage.values})\n\nmean_missing_percentage_steps = results_df['Missing Percentage (%)'].mean()\nprint(\"mean_missing_percentage_steps : \", mean_missing_percentage_steps)\nresults_df = results_df.sort_values(by='Missing Percentage (%)', ascending=False)\nprint(results_df.head())\n\nplt.figure(figsize=(10, 6))\nplt.bar(results_df['Column'], results_df['Missing Percentage (%)'], color='mediumseagreen')\nplt.xlabel('Column Name')\nplt.ylabel('Missing Percentage (%)')\nplt.title('Missing Values Percentage by Column')\nplt.xticks(rotation=90, fontsize=5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:37:10.174667Z","iopub.execute_input":"2024-10-26T09:37:10.176718Z","iopub.status.idle":"2024-10-26T09:37:11.314296Z","shell.execute_reply.started":"2024-10-26T09:37:10.176606Z","shell.execute_reply":"2024-10-26T09:37:11.312818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Calories burnt missing values","metadata":{}},{"cell_type":"code","source":"# get the missing values column wise and plot them\ncals_columns = train_csv.filter(regex='^cals')\nmissing_values = cals_columns.isnull().sum()\ntotal_rows = cals_columns.shape[0]\nmissing_percentage = (cals_columns.isnull().mean() * 100)\n\nresults_df = pd.DataFrame({'Column': missing_values.index,\n                          'Missing Values': missing_values.values,\n                          'Missing Percentage (%)': missing_percentage.values})\n\nmean_missing_percentage_cals = results_df['Missing Percentage (%)'].mean()\nprint(\"mean_missing_percentage_cals : \", mean_missing_percentage_cals)\nresults_df = results_df.sort_values(by='Missing Percentage (%)', ascending=False)\nprint(results_df.head())\n\nplt.figure(figsize=(10, 6))\nplt.bar(results_df['Column'], results_df['Missing Percentage (%)'], color='xkcd:salmon')\nplt.xlabel('Column Name')\nplt.ylabel('Missing Percentage (%)')\nplt.title('Missing Values Percentage by Column')\nplt.xticks(rotation=90, fontsize=5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:40:19.191683Z","iopub.execute_input":"2024-10-26T09:40:19.192145Z","iopub.status.idle":"2024-10-26T09:40:20.103036Z","shell.execute_reply.started":"2024-10-26T09:40:19.192104Z","shell.execute_reply":"2024-10-26T09:40:20.101919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Activities missing values","metadata":{}},{"cell_type":"code","source":"# get the missing values column wise and plot them\nactivity_columns = train_csv.filter(regex='^activity')\nmissing_values = activity_columns.isnull().sum()\ntotal_rows = activity_columns.shape[0]\nmissing_percentage = (activity_columns.isnull().mean() * 100)\n\nresults_df = pd.DataFrame({'Column': missing_values.index,\n                          'Missing Values': missing_values.values,\n                          'Missing Percentage (%)': missing_percentage.values})\n\nmean_missing_percentage_activity = results_df['Missing Percentage (%)'].mean()\nprint(\"mean_missing_percentage_activity : \", mean_missing_percentage_activity)\nresults_df = results_df.sort_values(by='Missing Percentage (%)', ascending=False)\nprint(results_df.head())\n\nplt.figure(figsize=(10, 6))\nplt.bar(results_df['Column'], results_df['Missing Percentage (%)'], color='xkcd:salmon')\nplt.xlabel('Column Name')\nplt.ylabel('Missing Percentage (%)')\nplt.title('Missing Values Percentage by Column')\nplt.xticks(rotation=90, fontsize=5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:42:11.278782Z","iopub.execute_input":"2024-10-26T09:42:11.279243Z","iopub.status.idle":"2024-10-26T09:42:13.591605Z","shell.execute_reply.started":"2024-10-26T09:42:11.279204Z","shell.execute_reply":"2024-10-26T09:42:13.590231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing = [mean_missing_values_bg, mean_missing_percentage_insulin, mean_missing_percentage_carbs, mean_missing_percentage_heart_rate, \n          mean_missing_percentage_steps, mean_missing_percentage_cals, mean_missing_percentage_activity]\nnames = [\"glucose\", \"insulin\", \"carbs\", \"heart-rate\", \"steps\", \"calories\", \"activity\"]","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:46:30.555480Z","iopub.execute_input":"2024-10-26T09:46:30.555943Z","iopub.status.idle":"2024-10-26T09:46:30.562647Z","shell.execute_reply.started":"2024-10-26T09:46:30.555901Z","shell.execute_reply":"2024-10-26T09:46:30.561171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pie chart\nplt.figure(figsize=(8, 8))\nplt.pie(\n    missing,\n    labels=names,\n    autopct='%1.1f%%',  # Display percentage on each slice\n    startangle=140,     # Starting angle for the chart\n    colors=plt.cm.Paired.colors  # Optional color scheme\n)\nplt.title('Mean Missing Percentage by Data Type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:46:32.737763Z","iopub.execute_input":"2024-10-26T09:46:32.738198Z","iopub.status.idle":"2024-10-26T09:46:33.000421Z","shell.execute_reply.started":"2024-10-26T09:46:32.738159Z","shell.execute_reply":"2024-10-26T09:46:32.999202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the Pie chart we can see that the data is etremly sparse with so many missing values. This might be the case why my model is not performing as it should be. It might be due to underre","metadata":{}}]}