{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61145,"databundleVersionId":7047669,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Applications","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"This notebook has to be created through the competition","metadata":{}},{"cell_type":"markdown","source":"## Table of Contents\n* [Read data](#Read-data)\n* [Data pre-processing](#Data-pre-processing)\n* [Map of all weather stations that measure air temperature](#Map-of-all-weather-stations-that-measure-air-temperature)\n* [What's next?](#What's-next)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport math\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:21:15.701407Z","iopub.execute_input":"2024-03-14T07:21:15.701834Z","iopub.status.idle":"2024-03-14T07:21:16.639475Z","shell.execute_reply.started":"2024-03-14T07:21:15.701786Z","shell.execute_reply":"2024-03-14T07:21:16.638506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], \n                       keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                       date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0],\n                          keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                          date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:21:16.642686Z","iopub.execute_input":"2024-03-14T07:21:16.643093Z","iopub.status.idle":"2024-03-14T07:23:17.077117Z","shell.execute_reply.started":"2024-03-14T07:21:16.643055Z","shell.execute_reply":"2024-03-14T07:23:17.075977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:17.078383Z","iopub.execute_input":"2024-03-14T07:23:17.078704Z","iopub.status.idle":"2024-03-14T07:23:17.090905Z","shell.execute_reply.started":"2024-03-14T07:23:17.078677Z","shell.execute_reply":"2024-03-14T07:23:17.089860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data pre-processing\n\n### Check dates\n\nWhen checking the last date of different kinds of dataframes it becomes clear that the watch survey responses only last until Jan 21, 2023, while the weather data lasts until July 3, 2023.","metadata":{}},{"cell_type":"code","source":"print(df_train.index.max())\nprint(df_solution.index.max())\nprint(df_rain.index.max())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:17.093604Z","iopub.execute_input":"2024-03-14T07:23:17.093951Z","iopub.status.idle":"2024-03-14T07:23:17.112249Z","shell.execute_reply.started":"2024-03-14T07:23:17.093926Z","shell.execute_reply":"2024-03-14T07:23:17.111358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't really need the weather data after Jan 21, 2023, so we might just as well get rid of it.","metadata":{}},{"cell_type":"code","source":"df_rain = df_rain[df_rain.index<'2023-01-22']\ndf_temperature = df_temperature[df_temperature.index<'2023-01-22']\ndf_humidity = df_humidity[df_humidity.index<'2023-01-22']\ndf_wind_speed = df_wind_speed[df_wind_speed.index<'2023-01-22']\ndf_wind_direction = df_wind_direction[df_wind_direction.index<'2023-01-22']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:17.113283Z","iopub.execute_input":"2024-03-14T07:23:17.113585Z","iopub.status.idle":"2024-03-14T07:23:17.150925Z","shell.execute_reply.started":"2024-03-14T07:23:17.113561Z","shell.execute_reply":"2024-03-14T07:23:17.149784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check weather data","metadata":{}},{"cell_type":"code","source":"# Show new shortened air temperature dataframe\ndf_temperature.describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:17.152272Z","iopub.execute_input":"2024-03-14T07:23:17.152648Z","iopub.status.idle":"2024-03-14T07:23:17.286692Z","shell.execute_reply.started":"2024-03-14T07:23:17.152614Z","shell.execute_reply":"2024-03-14T07:23:17.285550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like there is no temperature data available in before Jan 21, 2023 for weather station S117. So, we might just as well remove that column","metadata":{}},{"cell_type":"code","source":"df_temperature = df_temperature.drop(columns=['S117'])\n# Get dimensions of the air temperature data\ndf_temperature.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:17.287772Z","iopub.execute_input":"2024-03-14T07:23:17.288044Z","iopub.status.idle":"2024-03-14T07:23:17.297766Z","shell.execute_reply.started":"2024-03-14T07:23:17.288021Z","shell.execute_reply":"2024-03-14T07:23:17.296766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The weather dataframes are rather large. This can make the plotting of the data slow and the output images large. To make it a bit lighter, we are going to resample the data hourly.","metadata":{"execution":{"iopub.status.busy":"2024-03-12T21:09:04.412698Z","iopub.execute_input":"2024-03-12T21:09:04.413286Z","iopub.status.idle":"2024-03-12T21:09:04.419237Z","shell.execute_reply.started":"2024-03-12T21:09:04.413253Z","shell.execute_reply":"2024-03-12T21:09:04.41811Z"}}},{"cell_type":"code","source":"# Resample weather data\ndf_temperature_average = df_temperature.resample('60min').mean()\ndf_rain_sum = df_rain.resample('60min').sum() # note: for rain the data is aggregated with a sum instead of the mean\ndf_humidity_average = df_humidity.resample('60min').mean()\ndf_wind_speed_average = df_wind_speed.resample('60min').mean()\ndf_wind_direction_average = df_wind_direction.resample('60min').mean()\n\n# Check size of resulting dataframe\ndf_temperature_average.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:17.299157Z","iopub.execute_input":"2024-03-14T07:23:17.299831Z","iopub.status.idle":"2024-03-14T07:23:17.378807Z","shell.execute_reply.started":"2024-03-14T07:23:17.299803Z","shell.execute_reply":"2024-03-14T07:23:17.377753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Map watch survey questions to question ID\n","metadata":{}},{"cell_type":"code","source":"ws_questions = {\"q_noise_nearby\": \"Noise distractions nearby? (without earphones)\",\n                \"q_noise_kind\": \"What kind of noise?\",\n                \"q_earphones\": \"Wearing earphones?\",\n                \"q_thermal_preference\": \"Thermally, what do you prefer now?\",\n                \"q_location\": \"Where are you?\",\n                \"q_location_office\": \"What kind of office?\",\n                \"q_location_transport\": \"What kind of transport?\",\n                \"q_alone_group\": \"Alone or in a group?\",\n                \"q_activity_category_alone\": \"Category of activity? (alone)\",\n                \"q_activity_category_group\": \"Category of activity? (group)\"}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:17.380012Z","iopub.execute_input":"2024-03-14T07:23:17.380357Z","iopub.status.idle":"2024-03-14T07:23:17.385840Z","shell.execute_reply.started":"2024-03-14T07:23:17.380317Z","shell.execute_reply":"2024-03-14T07:23:17.384864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot weather data\n\n### Plot weather weather data time-series¶\n\nBelow we are going to plot the air temperature data from all weather stations. I used the Matplotlib method plt.plot() instead of sns.lineplot(). The reason is that plt.plot() shows the gaps in the data, while sns.lineplot() draws straight lines accross the gaps.\n","metadata":{}},{"cell_type":"code","source":"cols_num = 1\nrows_num = len(df_temperature_average.columns)\ncol_current = 0\nrow_current = 0 \nfig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\nplt.subplots_adjust(hspace = 0.5)\n\nfor i, col in enumerate(df_temperature_average.columns):\n    row_current = i\n    axs[row_current].plot(df_temperature_average.index, df_temperature_average[col])\n    station_name = df_stations[df_stations['id']==col]['name'].values[0]\n    axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n    axs[row_current].set_ylabel('Air Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:17.389700Z","iopub.execute_input":"2024-03-14T07:23:17.389975Z","iopub.status.idle":"2024-03-14T07:23:23.652196Z","shell.execute_reply.started":"2024-03-14T07:23:17.389953Z","shell.execute_reply":"2024-03-14T07:23:23.650900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since, we are going to plot more weather data, we might just as well wrap the above code for plotting in to a function.","metadata":{}},{"cell_type":"code","source":"# Function that plots weather data fromm all weather stations\ndef plot_weather(df, ylabel):\n    cols_num = 1\n    rows_num = len(df.columns)\n    col_current = 0\n    row_current = 0 \n    fig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\n    plt.subplots_adjust(hspace = 0.5)\n\n    for i, col in enumerate(df.columns):\n        row_current = i\n        axs[row_current].plot(df.index, df[col])\n        station_name = df_stations[df_stations['id']==col]['name'].values[0]\n        axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n        axs[row_current].set_ylabel(ylabel)\n    \n    return fig, axs","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:23.653723Z","iopub.execute_input":"2024-03-14T07:23:23.654534Z","iopub.status.idle":"2024-03-14T07:23:23.663312Z","shell.execute_reply.started":"2024-03-14T07:23:23.654498Z","shell.execute_reply":"2024-03-14T07:23:23.662361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Temperature","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_temperature_average, 'Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:23.664684Z","iopub.execute_input":"2024-03-14T07:23:23.665076Z","iopub.status.idle":"2024-03-14T07:23:29.831949Z","shell.execute_reply.started":"2024-03-14T07:23:23.665047Z","shell.execute_reply":"2024-03-14T07:23:29.830898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Rainfall","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_rain_sum, 'Rainfall [mm/h]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:29.833181Z","iopub.execute_input":"2024-03-14T07:23:29.833496Z","iopub.status.idle":"2024-03-14T07:23:56.067754Z","shell.execute_reply.started":"2024-03-14T07:23:29.833468Z","shell.execute_reply":"2024-03-14T07:23:56.066577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relative humidity","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_humidity_average, 'Relative humidity [%]')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:23:56.069080Z","iopub.execute_input":"2024-03-14T07:23:56.069415Z","iopub.status.idle":"2024-03-14T07:24:02.339832Z","shell.execute_reply.started":"2024-03-14T07:23:56.069387Z","shell.execute_reply":"2024-03-14T07:24:02.338374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Map of all weather stations that measure air temperature","metadata":{}},{"cell_type":"code","source":"# Have a look at the weather station meta data\ndf_stations.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.341254Z","iopub.execute_input":"2024-03-14T07:24:02.341665Z","iopub.status.idle":"2024-03-14T07:24:02.355874Z","shell.execute_reply.started":"2024-03-14T07:24:02.341633Z","shell.execute_reply":"2024-03-14T07:24:02.354844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.357398Z","iopub.execute_input":"2024-03-14T07:24:02.357774Z","iopub.status.idle":"2024-03-14T07:24:02.369871Z","shell.execute_reply.started":"2024-03-14T07:24:02.357743Z","shell.execute_reply":"2024-03-14T07:24:02.368362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a look at the air temperature data\ndf_temperature.columns","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.371673Z","iopub.execute_input":"2024-03-14T07:24:02.372116Z","iopub.status.idle":"2024-03-14T07:24:02.382036Z","shell.execute_reply.started":"2024-03-14T07:24:02.372073Z","shell.execute_reply":"2024-03-14T07:24:02.381104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, the list of all weather stations is much longer then the list with weather stations that can measure air temperature. Hence, we need to pre-process the data a bit and only pick the weather stations that can measure temperature data.","metadata":{}},{"cell_type":"code","source":"# Initialize a new dataframe for creating the map\ndf_map = pd.DataFrame(columns=['latitude', 'longitude', 'text'])\n\n\n# Go through each column of the temperature dataframe\nfor col in df_temperature.columns:\n    # Get the weather station name based on the station id\n    station_name = df_stations.loc[df_stations.id==col].name.values[0]\n    \n    # Create a new row, with latitude, longitude, hover text, and marker color as columns\n    df_row = pd.DataFrame([{'latitude': df_stations.loc[df_stations.id==col].latitude.values[0], \n                            'longitude': df_stations.loc[df_stations.id==col].longitude.values[0], \n                            'text': f'Station ID: {col} <br>Station name: {station_name} <br>Measurement: Rain',\n                            'color': 'rgb(0, 128, 255)'}])\n    \n    # Append new row to map dataframe\n    df_map = pd.concat([df_map, df_row])\n    \ndf_map.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.383340Z","iopub.execute_input":"2024-03-14T07:24:02.383702Z","iopub.status.idle":"2024-03-14T07:24:02.443058Z","shell.execute_reply.started":"2024-03-14T07:24:02.383668Z","shell.execute_reply":"2024-03-14T07:24:02.442044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pre-processing for the maps is done. Now, we can go and actually create the map chart.","metadata":{}},{"cell_type":"code","source":"# Create trace and add markers to map\ntrace = go.Scattermapbox(lat=df_map.latitude, \n                         lon=df_map.longitude,\n                         mode='markers',\n                         marker=dict(size=10,\n                                     color=df_map['color'],\n                                     opacity=0.8),\n                         hovertemplate = \"\",\n                         text=df_map['text'])\n\n# Define the layout for the map\nlayout = go.Layout(\n    margin={\"l\": 0, \"r\": 0, \"t\": 0, \"b\": 0},\n    mapbox=dict(\n        style='carto-positron', # Alternative: 'open-street-map',\n        center=dict(lat=df_map.latitude.median(),\n                    lon=df_map.longitude.median()),\n        zoom=10,\n        ),\n    height=700, width=900)\n\n# Create figure\nfig = go.Figure(data=[trace], layout=layout)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.444253Z","iopub.execute_input":"2024-03-14T07:24:02.444615Z","iopub.status.idle":"2024-03-14T07:24:02.663327Z","shell.execute_reply.started":"2024-03-14T07:24:02.444588Z","shell.execute_reply":"2024-03-14T07:24:02.662534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, you can hovering the cursor over the markers will reveal some meta information about the weather station.","metadata":{}},{"cell_type":"markdown","source":"## What's next?","metadata":{}},{"cell_type":"markdown","source":"How do I continue this? Where to start?!\n\nGO TO THE COMPETITION SITE!\n\n[https://www.kaggle.com/competitions/cool-quiet-city-competition](https://www.kaggle.com/competitions/cool-quiet-city-competition)","metadata":{}},{"cell_type":"markdown","source":"## Interpreting the code\n\n- I tried to interpret the code and optimize it, but I had some problems.","metadata":{}},{"cell_type":"markdown","source":"## Create a dataframe for the wind direction\n\nThis transformation aids in the analysis of wind direction by representing it in a way that accounts for its circular nature.","metadata":{}},{"cell_type":"code","source":"df_wind_direction_sin = df_wind_direction.copy(deep=True)\ndf_wind_direction_cos = df_wind_direction.copy(deep=True)\nfor s in df_wind_direction.columns:\n    df_wind_direction_sin[s] = np.sin(np.pi/180*df_wind_direction_sin[s])\n    df_wind_direction_cos[s] = np.cos(np.pi/180*df_wind_direction_cos[s])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.664618Z","iopub.execute_input":"2024-03-14T07:24:02.664920Z","iopub.status.idle":"2024-03-14T07:24:02.769750Z","shell.execute_reply.started":"2024-03-14T07:24:02.664892Z","shell.execute_reply":"2024-03-14T07:24:02.768618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Distribution of the transformed wind direction data by examining its central tendency and variability.","metadata":{}},{"cell_type":"code","source":"pd.DataFrame(df_wind_direction_sin[s]).describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.770921Z","iopub.execute_input":"2024-03-14T07:24:02.771227Z","iopub.status.idle":"2024-03-14T07:24:02.792322Z","shell.execute_reply.started":"2024-03-14T07:24:02.771200Z","shell.execute_reply":"2024-03-14T07:24:02.791273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(df_wind_direction_cos[s]).describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.793386Z","iopub.execute_input":"2024-03-14T07:24:02.793654Z","iopub.status.idle":"2024-03-14T07:24:02.812977Z","shell.execute_reply.started":"2024-03-14T07:24:02.793631Z","shell.execute_reply":"2024-03-14T07:24:02.811992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge train and test data in a single dataframe","metadata":{}},{"cell_type":"code","source":"# Tagging the data sources in each DataFrame before combining\ndf_train['source']='train'\ndf_solution['source']='solution'\n\n# Concatenating the training and solution data into a single DataFrame\ndf_all = pd.concat([df_train, df_solution])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:02.814127Z","iopub.execute_input":"2024-03-14T07:24:02.814453Z","iopub.status.idle":"2024-03-14T07:24:03.516382Z","shell.execute_reply.started":"2024-03-14T07:24:02.814426Z","shell.execute_reply":"2024-03-14T07:24:03.515432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_all.shape)\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:03.517784Z","iopub.execute_input":"2024-03-14T07:24:03.518553Z","iopub.status.idle":"2024-03-14T07:24:03.551894Z","shell.execute_reply.started":"2024-03-14T07:24:03.518513Z","shell.execute_reply":"2024-03-14T07:24:03.550825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One hot encoding for categorical features, and dropping the categorical variables","metadata":{}},{"cell_type":"markdown","source":"This code segments and cleans survey data by transforming categorical answers into boolean columns and then removing the original columns.\n","metadata":{}},{"cell_type":"code","source":"# Creating boolean columns based on respondents' answers to location and activity questions for detailed analysis\ndf_all['q_Indoor'] = df_all['q_location'].str.contains(\"Indoor\")\ndf_all['q_Class'] = df_all['q_location'].isin(['Indoor - Class'])\ndf_all['q_Home'] = df_all['q_location'].isin(['Indoor - Home'])\ndf_all['q_Office'] = df_all['q_location'].isin(['Indoor - Office'])\ndf_all['q_Indoor_Other'] = df_all['q_location'].isin(['Indoor - Other'])\ndf_all['q_Outdoor'] = df_all['q_location'].isin(['Outdoor'])\ndf_all['q_Transportation'] = df_all['q_location'].isin(['Transportation'])\ndf_all['q_alone_group'] = df_all['q_alone_group'].isin(['Alone'])\ndf_all['q_Bus'] = df_all['q_location_transport'].isin(['Bus'])\ndf_all['q_Car'] = df_all['q_location_transport'].isin(['Car'])\ndf_all['q_Taxi'] = df_all['q_location_transport'].isin(['Taxi'])\ndf_all['q_Train'] = df_all['q_location_transport'].isin(['Train'])\ndf_all['q_Transport_Other'] = df_all['q_location_transport'].isin(['Other'])\ndf_all['q_Focus'] = df_all['q_activity_category_alone'].isin(['Focus'])\ndf_all['q_Leisure'] = df_all['q_activity_category_alone'].isin(['Leisure'])\ndf_all['q_Alone_Activity_Other'] = df_all['q_activity_category_alone'].isin(['Other'])\ndf_all['q_Collaborate'] = df_all['q_activity_category_group'].isin(['Collaborate'])\ndf_all['q_Learn'] = df_all['q_activity_category_group'].isin(['Learn'])\ndf_all['q_Socialize'] = df_all['q_activity_category_group'].isin(['Socialize'])\ndf_all['q_Group_Activity_Other'] = df_all['q_activity_category_group'].isin(['Other'])\ndf_all['q_Conference room'] = df_all['q_location_office'].isin(['Conference room'])\ndf_all['q_Cubicles'] = df_all['q_location_office'].isin(['Cubicles'])\ndf_all['q_Individual'] = df_all['q_location_office'].isin(['Individual'])\ndf_all['q_Large open plan'] = df_all['q_location_office'].isin(['Large open plan'])\ndf_all['q_Not applicable'] = df_all['q_location_office'].isin(['Not applicable'])\ndf_all['q_Small shared'] = df_all['q_location_office'].isin(['Small shared'])\n\n# Dropping the original categorical columns after boolean transformation\ndf_all = df_all.drop(columns=['q_location','q_location_transport','q_activity_category_alone','q_activity_category_group','q_location_office'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:03.553857Z","iopub.execute_input":"2024-03-14T07:24:03.554343Z","iopub.status.idle":"2024-03-14T07:24:06.085930Z","shell.execute_reply.started":"2024-03-14T07:24:03.554279Z","shell.execute_reply":"2024-03-14T07:24:06.085012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate the difference in seconds between the survey start and submission times","metadata":{}},{"cell_type":"code","source":"time_difference = (df_all.index - df_all['ws_timestamp_start']).dt.total_seconds()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:06.087172Z","iopub.execute_input":"2024-03-14T07:24:06.087501Z","iopub.status.idle":"2024-03-14T07:24:06.139330Z","shell.execute_reply.started":"2024-03-14T07:24:06.087472Z","shell.execute_reply":"2024-03-14T07:24:06.138294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Handle potential log of zero or negative values by replacing them with a small positive value before taking the log","metadata":{}},{"cell_type":"code","source":"df_all['duration'] = np.log(time_difference.clip(lower=1))","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:06.140724Z","iopub.execute_input":"2024-03-14T07:24:06.141130Z","iopub.status.idle":"2024-03-14T07:24:06.172797Z","shell.execute_reply.started":"2024-03-14T07:24:06.141091Z","shell.execute_reply":"2024-03-14T07:24:06.171815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compute temporary variables for hour and minute to avoid repeated calculations","metadata":{}},{"cell_type":"code","source":"hours_in_minutes = df_all.index.hour * 60\nminutes = df_all.index.minute\ntime_in_minutes = hours_in_minutes + minutes","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:06.179650Z","iopub.execute_input":"2024-03-14T07:24:06.179975Z","iopub.status.idle":"2024-03-14T07:24:06.358797Z","shell.execute_reply.started":"2024-03-14T07:24:06.179948Z","shell.execute_reply":"2024-03-14T07:24:06.357821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compute day and time features","metadata":{}},{"cell_type":"code","source":"angle_in_radians = np.deg2rad(time_in_minutes / 4)  # Dividing by 4 converts time to degrees, then to radians\ndf_all['time_of_day_cos'] = np.cos(angle_in_radians)\ndf_all['time_of_day_sin'] = np.sin(angle_in_radians)\ndf_all['time_of_day'] = time_in_minutes\ndf_all['day_of_week'] = df_all.index.dayofweek","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:06.360210Z","iopub.execute_input":"2024-03-14T07:24:06.360569Z","iopub.status.idle":"2024-03-14T07:24:06.571783Z","shell.execute_reply.started":"2024-03-14T07:24:06.360539Z","shell.execute_reply":"2024-03-14T07:24:06.570633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compute aggreagates to the features provided by Apple HealthKit","metadata":{}},{"cell_type":"code","source":"#######\n# I tried to optimize the code using a function but didn't work, this code give me error in the execution of the next cell.\n# Error:\n'''\nKeyError                                  Traceback (most recent call last)\nFile /opt/conda/lib/python3.10/site-packages/pandas/core/indexes/base.py:3802, in Index.get_loc(self, key)\n   3801 try:\n-> 3802     return self._engine.get_loc(casted_key)\n   3803 except KeyError as err:\n\nFile index.pyx:153, in pandas._libs.index.IndexEngine.get_loc()\n\nFile index.pyx:182, in pandas._libs.index.IndexEngine.get_loc()\n\nFile pandas/_libs/hashtable_class_helper.pxi:7081, in pandas._libs.hashtable.PyObjectHashTable.get_item()\n\nFile pandas/_libs/hashtable_class_helper.pxi:7089, in pandas._libs.hashtable.PyObjectHashTable.get_item()\n\nKeyError: 'mean_resting_heart_rate'\n\nThe above exception was the direct cause of the following exception:\n\nKeyError                                  Traceback (most recent call last)\nCell In[41], line 11\n      8 mean_feature = f'mean_{feature}'\n      9 std_feature = f'std_{feature}'\n---> 11 df_all[ts_feature] = (df_all[ts_feature] - df_all[mean_feature]) / df_all[std_feature]\n\nFile /opt/conda/lib/python3.10/site-packages/pandas/core/frame.py:4090, in DataFrame.__getitem__(self, key)\n   4088 if self.columns.nlevels > 1:\n   4089     return self._getitem_multilevel(key)\n-> 4090 indexer = self.columns.get_loc(key)\n   4091 if is_integer(indexer):\n   4092     indexer = [indexer]\n\nFile /opt/conda/lib/python3.10/site-packages/pandas/core/indexes/base.py:3809, in Index.get_loc(self, key)\n   3804     if isinstance(casted_key, slice) or (\n   3805         isinstance(casted_key, abc.Iterable)\n   3806         and any(isinstance(x, slice) for x in casted_key)\n   3807     ):\n   3808         raise InvalidIndexError(key)\n-> 3809     raise KeyError(key) from err\n   3810 except TypeError:\n   3811     # If we have a listlike key, _check_indexing_error will raise\n   3812     #  InvalidIndexError. Otherwise we fall through and re-raise\n   3813     #  the TypeError.\n   3814     self._check_indexing_error(key)\n\nKeyError: 'mean_resting_heart_rate'\n'''\n#######\n# Code i tried to use\n# Define the columns for which you want to calculate the statistics\n'''\ncolumns_to_transform = [\n    'ts_resting_heart_rate',\n    'ts_audio_exposure_environment',\n    'ts_oxygen_saturation',\n    'ts_stand_time',\n    'ts_heart_rate',\n    'ts_step_count',\n    'ts_walking_distance'\n]\n\n# Use a single groupby operation and calculate all the required statistics\nfor column in columns_to_transform:\n    grouped = df_all.groupby('id_participant')[column]\n    df_all[f'med_{column}'] = grouped.transform('median')\n    df_all[f'mean_{column}'] = grouped.transform('mean')\n    df_all[f'std_{column}'] = grouped.transform('std')\n'''\n\n# Compute and add the median of health metrics for each participant\ndf_all['med_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('median')\ndf_all['med_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('median')\ndf_all['med_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('median')\ndf_all['med_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('median')\ndf_all['med_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('median')\ndf_all['med_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('median')\ndf_all['med_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('median')\n\n# Compute and add the mean of health metrics for each participant\ndf_all['mean_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('mean')\ndf_all['mean_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('mean')\ndf_all['mean_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('mean')\ndf_all['mean_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('mean')\ndf_all['mean_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('mean')\ndf_all['mean_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('mean')\ndf_all['mean_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('mean')\n\n# Compute and add the standard deviation of health metrics for each participant\ndf_all['std_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('std')\ndf_all['std_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('std')\ndf_all['std_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('std')\ndf_all['std_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('std')\ndf_all['std_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('std')\ndf_all['std_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('std')\ndf_all['std_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('std')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:06.573702Z","iopub.execute_input":"2024-03-14T07:24:06.574131Z","iopub.status.idle":"2024-03-14T07:24:12.064087Z","shell.execute_reply.started":"2024-03-14T07:24:06.574092Z","shell.execute_reply":"2024-03-14T07:24:12.063059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalize the features provided by Apple HealthKit\nStandardizes health metric features in `df_all` by subtracting the mean and dividing by the standard deviation for each participant.\n\n- I add a function to optimize and make the code more maintainable","metadata":{}},{"cell_type":"code","source":"# Standardize specified features for analysis\nfeatures_to_standardize = [\n    'resting_heart_rate', 'audio_exposure_environment', 'oxygen_saturation',\n    'stand_time', 'heart_rate', 'step_count', 'walking_distance'\n]\n\nfor feature in features_to_standardize:\n    ts_feature = f'ts_{feature}'  # Time series data for the feature\n    mean_feature = f'mean_{feature}'  # Mean of the feature\n    std_feature = f'std_{feature}'  # Standard deviation of the feature\n\n    # Standardization formula: (value - mean) / std deviation\n    df_all[ts_feature] = (df_all[ts_feature] - df_all[mean_feature]) / df_all[std_feature]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:12.065499Z","iopub.execute_input":"2024-03-14T07:24:12.065901Z","iopub.status.idle":"2024-03-14T07:24:12.152370Z","shell.execute_reply.started":"2024-03-14T07:24:12.065861Z","shell.execute_reply":"2024-03-14T07:24:12.151348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Interpolate the features provided by Apple HealthKit\n\nResamples health data within `df_all` for consistency, using linear interpolation to fill missing values for a list of specified health metrics.","metadata":{}},{"cell_type":"code","source":"# Modified function to resample multiple columns at once\ndef resample_health(df, col_names):\n    for col_name in col_names:\n        col_name_new = 'ws_' + col_name[3:]  # Assumes 'ts_' prefix to be replaced by 'ws_'\n        df[col_name_new] = df[col_name].interpolate(method='linear').bfill().ffill().mask(df['ws_survey_count'].isna())\n    return df\n\n# List of column names to be resampled\ncolumns_to_resample = [\n    'ts_heart_rate',\n    'ts_audio_exposure_environment',\n    'ts_oxygen_saturation',\n    'ts_resting_heart_rate',\n    'ts_stand_time',\n    'ts_step_count',\n    'ts_walking_distance'\n]\n\n# Group once and apply the resampling to all columns\ndf_all = df_all.groupby('id_participant', group_keys=False).apply(resample_health, columns_to_resample)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:12.153680Z","iopub.execute_input":"2024-03-14T07:24:12.154000Z","iopub.status.idle":"2024-03-14T07:24:30.227636Z","shell.execute_reply.started":"2024-03-14T07:24:12.153972Z","shell.execute_reply":"2024-03-14T07:24:30.226771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Interpolate weather data\n\nRetrieves weather data close to a specific timestamp and location by finding the nearest weather station and calculating the distance using the Haversine formula.","metadata":{}},{"cell_type":"code","source":"from datetime import datetime, timedelta\nfrom math import cos, asin, sqrt, pi\n\n# Retrieve weather data based on proximity in time and distance to weather stations\ndef get_weather_data(df_weather_data, lat, lon, timestamp):    \n    # Get timestamp with no seconds or microseconds\n    timestamp = timestamp.replace(second=0, microsecond=0)\n    \n    # Get weather data for timestamp above\n    weather_data_row = df_weather_data[(df_weather_data.index >= timestamp- timedelta(hours=0, minutes=10)) & (df_weather_data.index <= timestamp + timedelta(hours=0, minutes=10))]\n\n    # If no weather data is available for timestamp above, get weather data for next minute\n    if (weather_data_row.empty):\n        timestamp = timestamp + pd.Timedelta(minutes=1)\n        weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n        # If there is still no weather data available, return NaN\n        if (weather_data_row.empty):\n            timestamp = timestamp - 2 * pd.Timedelta(minutes=1)\n            weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n            if (weather_data_row.empty):\n                return np.NaN\n\n    # Get weather station ids from weather stations with with data\n    station_ids_with_data = weather_data_row.columns[weather_data_row.count() > 0].tolist()\n\n    # Get weather stations with data\n    df_stations2 = df_stations[df_stations.id.isin(station_ids_with_data)]\n\n    # Compute distance to each weather station\n    df_stations2['distance'] = df_stations2.apply(lambda x: distance(x['latitude'], x['longitude'], lat, lon), axis=1)\n\n    # Get nearest weather station\n    station_id_nearest = df_stations2[df_stations2.distance==df_stations2.distance.min()].id.values[0]\n\n    # Retrun NaN for cases where the nearest weather station is more than 10km away\n    if df_stations2.distance.min()>10:\n        return np.NaN\n\n    # Return weather data from nearest weather station\n    return np.nanmean(weather_data_row[station_id_nearest])\n\n# Function to compute distnace on spherical earth\ndef distance(lat1, lon1, lat2, lon2):\n    r = 6371 # km\n    p = pi / 180\n\n    a = 0.5 - cos((lat2-lat1)*p)/2 + cos(lat1*p) * cos(lat2*p) * (1-cos((lon2-lon1)*p))/2\n    return 2 * r * asin(sqrt(a))","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:30.228994Z","iopub.execute_input":"2024-03-14T07:24:30.229289Z","iopub.status.idle":"2024-03-14T07:24:30.242716Z","shell.execute_reply.started":"2024-03-14T07:24:30.229263Z","shell.execute_reply":"2024-03-14T07:24:30.241298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Populates the dataset with weather conditions like rainfall, temperature, humidity, and wind speed/direction by applying a function to fetch data from the nearest weather station.","metadata":{}},{"cell_type":"code","source":"# Reset index to access the 'time' column for applying the weather data retrieval function\ndf_all = df_all.reset_index()\n\n# Apply the function to fetch weather data for each condition only for rows with survey counts\ndf_all['ws_rainfall'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_rain, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_air_temperature'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_temperature, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_relative_humidity'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_humidity, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_speed'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_speed, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_sin'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_sin, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_cos'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_cos, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\n# Set the 'time' column back as the index of df_all\ndf_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:24:30.244113Z","iopub.execute_input":"2024-03-14T07:24:30.244580Z","iopub.status.idle":"2024-03-14T07:27:48.423276Z","shell.execute_reply.started":"2024-03-14T07:24:30.244544Z","shell.execute_reply":"2024-03-14T07:27:48.422251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Remove unused rows and columns \n\nFilters the dataset to retain only watch survey responses and removes unnecessary columns to streamline the data for analysis.","metadata":{}},{"cell_type":"code","source":"# Remove all rows that are not a watch surve response\ndf_all = df_all[df_all.ws_survey_count.notna()]\n\n# Remove some timestamp columns\ndf_all = df_all.drop(columns=['ws_timestamp_location', 'ws_timestamp_start'])\n\n# Remove columns of the original health data those are no longer needed since we are going to use the interpolated values\ndf_all = df_all.drop(columns=['ts_heart_rate',\n                              'ts_audio_exposure_environment',\n                              'ts_oxygen_saturation',\n                              'ts_resting_heart_rate',\n                              'ts_stand_time', \n                              'ts_step_count',\n                              'ts_walking_distance'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:27:48.424512Z","iopub.execute_input":"2024-03-14T07:27:48.424829Z","iopub.status.idle":"2024-03-14T07:27:48.505015Z","shell.execute_reply.started":"2024-03-14T07:27:48.424802Z","shell.execute_reply":"2024-03-14T07:27:48.503813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"own_earphone and own_noise_cancelling noise cancelling are two features that represent whether a user actually own an earphone or a noise cancelling headset, this information is infered by the user previous answers.\n\nAssigns boolean flags to participants based on their usage of earphones or noise-cancelling devices, enhancing data with user preferences.","metadata":{}},{"cell_type":"code","source":"# Create a dictionary to count usage instances of earphone types by participant\nearphone_dict = df_all.groupby(['id_participant', 'q_earphones'])['duration'].count().to_dict()\n\n# Initialize columns to indicate ownership of earphones and noise-cancelling devices\ndf_all['own_earphones'] = 0\ndf_all['own_noise_cancelling'] = 0\n\n# Iterate through rows to set flags based on the earphone dictionary\nfor index, row in df_all.iterrows():\n    if (row['id_participant'], 'Earphones') in earphone_dict:\n        df_all.at[index, 'own_earphones'] = 1\n    if (row['id_participant'], 'Noise cancelling') in earphone_dict:\n        df_all.at[index, 'own_noise_cancelling'] = 1","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:27:48.506461Z","iopub.execute_input":"2024-03-14T07:27:48.506848Z","iopub.status.idle":"2024-03-14T07:27:50.613695Z","shell.execute_reply.started":"2024-03-14T07:27:48.506810Z","shell.execute_reply":"2024-03-14T07:27:50.612704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"update the features own_earphone and own_noise_cancelling by the fact that nobody uses noise cancelling in class since it defies the purpose of going to class, and the users who haven't used earphones in class, they won't start doing that.\n\nIdentifies participants who attended class without using earphones and updates their records to reflect this, ensuring accurate representation of earphone use.","metadata":{}},{"cell_type":"code","source":"# Identify participants using earphones in class and those who have been to class\nUse_earphone_in_class = set(df_all[(df_all['q_Class']) & (df_all['q_earphones'] == 'Earphones')]['id_participant'].unique())\nbeen_to_class = set(df_all[(df_all['q_Class']) & (df_all['source'] == 'train')]['id_participant'].unique())\n\n# Determine participants who have been to class but did not use earphones\ndiff = been_to_class.difference(Use_earphone_in_class)\n\n# Iterate through DataFrame to update earphone usage based on attendance and use\nfor index, row in df_all.iterrows():\n    if row['q_Class']:\n        df_all.at[index, 'own_noise_cancelling'] = 0\n        if row['id_participant'] in diff:\n            df_all.at[index, 'own_earphones'] = 0","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:27:50.615014Z","iopub.execute_input":"2024-03-14T07:27:50.615424Z","iopub.status.idle":"2024-03-14T07:27:51.354619Z","shell.execute_reply.started":"2024-03-14T07:27:50.615387Z","shell.execute_reply":"2024-03-14T07:27:51.353785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculates and records the number of occurrences at each unique location based on latitude and longitude, enriching the dataset with spatial frequency information.","metadata":{}},{"cell_type":"code","source":"# Initialize a column to count occurrences at each location\ndf_all['location_count'] = 0\n\n# Iterate through the DataFrame to count and assign the number of occurrences for each unique location\nfor index, row in df_all.iterrows():\n    df_all.at[index, 'location_count'] = len(df_all[(df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:27:51.355943Z","iopub.execute_input":"2024-03-14T07:27:51.356285Z","iopub.status.idle":"2024-03-14T07:28:03.286472Z","shell.execute_reply.started":"2024-03-14T07:27:51.356249Z","shell.execute_reply":"2024-03-14T07:28:03.285550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Assigns a probability to each row indicating the frequency of a participant's occurrence at a specific location, offering insights into location preferences.","metadata":{}},{"cell_type":"code","source":"# Initialize a column for the probability of being at a given location\ndf_all['location_prob'] = 0\n\n# Iterate through DataFrame to calculate and assign location probability for each participant\nfor index, row in df_all.iterrows():\n    temp = df_all[(df_all['id_participant'] == row['id_participant']) & \n                  (df_all['ws_latitude'] == row['ws_latitude']) & \n                  (df_all['ws_longitude'] == row['ws_longitude'])]\n    prob = len(temp) / len(df_all[df_all['id_participant'] == row['id_participant']])\n    df_all.at[index, 'location_prob'] = prob","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:28:03.287611Z","iopub.execute_input":"2024-03-14T07:28:03.287882Z","iopub.status.idle":"2024-03-14T07:28:56.989987Z","shell.execute_reply.started":"2024-03-14T07:28:03.287858Z","shell.execute_reply":"2024-03-14T07:28:56.989080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split the data","metadata":{}},{"cell_type":"code","source":"# Separte training and solution data again\ndf_train = df_all[df_all.source == 'train'].copy()\ndf_solution = df_all[df_all.source == 'solution'].copy()\n\n# Drop the helper columns ws_surve_count_2 and source again\ndf_train = df_train.drop(columns=['source'])\ndf_solution = df_solution.drop(columns=['source'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:28:56.991219Z","iopub.execute_input":"2024-03-14T07:28:56.991549Z","iopub.status.idle":"2024-03-14T07:28:57.082220Z","shell.execute_reply.started":"2024-03-14T07:28:56.991521Z","shell.execute_reply":"2024-03-14T07:28:57.081449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preparation\n\n - input and output values need to be numeric","metadata":{}},{"cell_type":"code","source":"# Find columns with values that are not numeric\ncat_vars = list(df_train.select_dtypes(include=['object']).columns)\n# encoding categorical variables\ndict_cat = {}\n\nfor var in cat_vars:\n    if var == 'id_participant': # no need to encode the participant ID\n        continue\n    # convert categorical variables into 'category' type\n    df_train[var] = df_train[var].astype('category')\n    df_solution[var] = df_solution[var].astype('category')\n    \n    # dictionary to keep track of the variable mapping: {variable-name: {number1: string1, number2: string2, ...}\n    dict_cat[var] = dict(enumerate(df_train[var].cat.categories))\n\n    # encode variables, missing values are encoded as -1\n    df_train[var] = df_train[var].cat.codes\n    df_solution[var] = df_solution[var].cat.codes","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:28:57.083279Z","iopub.execute_input":"2024-03-14T07:28:57.083548Z","iopub.status.idle":"2024-03-14T07:28:57.108270Z","shell.execute_reply.started":"2024-03-14T07:28:57.083525Z","shell.execute_reply":"2024-03-14T07:28:57.107426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Function to predict in any column","metadata":{}},{"cell_type":"markdown","source":"### Imports classes for constructing machine learning pipelines and various ensemble learning methods from the scikit-learn library.","metadata":{}},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier\nfrom sklearn.ensemble import VotingClassifier, StackingClassifier\n\nfrom sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.metrics import confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:40:36.931190Z","iopub.execute_input":"2024-03-14T07:40:36.931979Z","iopub.status.idle":"2024-03-14T07:40:36.938288Z","shell.execute_reply.started":"2024-03-14T07:40:36.931944Z","shell.execute_reply":"2024-03-14T07:40:36.937223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Function to predict using sklearn","metadata":{}},{"cell_type":"code","source":"def predict_any_column(column_name, df_all, df_solution):\n    # Remove rows where the predicting variable is missing\n    df_all = df_all[df_all[column_name] != -1]\n    \n    # Remove rows where the predicting variable is missing\n    df_all = df_all.drop(columns=['id_participant'])\n    df_solution = df_solution.drop(columns=['id_participant'])\n    \n    # Count classes in predicting variable\n    num_class = df_train[column_name].unique()\n    \n    # Break dataframe into variables and ground truth\n    y_all = df_all[column_name].to_numpy()\n    X_all = df_all.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)#.to_numpy()\n    cat_features = [f for f in cat_vars if f in X_all.columns]\n    X_all2 = X_all.to_numpy()\n    X_solution = df_solution.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)\n    X_solution2 = X_solution.to_numpy()\n    # Select features\n    #sel = SelectFromModel(XGBClassifier(num_class=num_class, objective='multi:softmax', enable_categorical=False))\n    #sel.fit(X_all,y_all)\n    #selected_feat= X_all.columns[(sel.get_support())]\n    arr = np.zeros((len(num_class),len(num_class)))\n    score = 0\n    print(num_class)\n    y_solution = np.zeros(shape=(X_solution.shape[0],len(num_class)))\n    y_train = np.zeros(shape=(X_all.shape[0],len(num_class)))\n    skf = StratifiedKFold(n_splits=5,random_state=1990,shuffle=True)\n    for i, (train_index, test_index) in enumerate(skf.split(X_all, y_all)):\n        # Initialize model\n        models = [          \n           ('catBoost',make_pipeline(CatBoostClassifier(iterations=1000, verbose=1000, random_seed=42,learning_rate=0.05))),          \n           ('lgbm',make_pipeline(lgb.LGBMClassifier(random_state=42))),   \n           ('XGB',make_pipeline(XGBClassifier(objective='multi:softmax',random_state=42)))     \n            \n        ]\n        voter = VotingClassifier(models, voting = 'soft')\n        # Predict and evaluate\n        voter.fit(X_all.iloc[train_index], y_all[train_index])\n        y_pred = voter.predict_proba(X_all.iloc[test_index]) \n        \n        score += accuracy_score(y_all[test_index], np.argmax(y_pred, axis=1))/5\n        arr = arr + confusion_matrix(y_all[test_index], np.argmax(y_pred, axis=1))\n\n        # Predict solution\n        y_solution += voter.predict_proba(X_solution) #+ clf2.predict_proba(X_solution2)\n    \n    print('Model accuracy score: {0:0.4f}'. format(score))\n    #print(arr)\n    \n    return np.argmax(y_solution, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:40:39.054781Z","iopub.execute_input":"2024-03-14T07:40:39.055168Z","iopub.status.idle":"2024-03-14T07:40:39.070042Z","shell.execute_reply.started":"2024-03-14T07:40:39.055138Z","shell.execute_reply":"2024-03-14T07:40:39.068941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a copy of the solution dataframe for submission\n","metadata":{}},{"cell_type":"code","source":"df_submission = df_solution.copy()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:40:41.610777Z","iopub.execute_input":"2024-03-14T07:40:41.611945Z","iopub.status.idle":"2024-03-14T07:40:41.618730Z","shell.execute_reply.started":"2024-03-14T07:40:41.611898Z","shell.execute_reply":"2024-03-14T07:40:41.617706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predictions","metadata":{}},{"cell_type":"markdown","source":"### Prediction q_noise_nearby","metadata":{}},{"cell_type":"code","source":"df_submission['q_noise_nearby'] = predict_any_column('q_noise_nearby', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:40:43.368761Z","iopub.execute_input":"2024-03-14T07:40:43.369678Z","iopub.status.idle":"2024-03-14T07:42:52.343889Z","shell.execute_reply.started":"2024-03-14T07:40:43.369639Z","shell.execute_reply":"2024-03-14T07:42:52.342864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_noise_nearby'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:42:58.630887Z","iopub.execute_input":"2024-03-14T07:42:58.631336Z","iopub.status.idle":"2024-03-14T07:42:58.645543Z","shell.execute_reply.started":"2024-03-14T07:42:58.631279Z","shell.execute_reply":"2024-03-14T07:42:58.644521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction q_noise_kind","metadata":{}},{"cell_type":"code","source":"df_submission['q_noise_kind'] = predict_any_column('q_noise_kind', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:43:12.257692Z","iopub.execute_input":"2024-03-14T07:43:12.258084Z","iopub.status.idle":"2024-03-14T07:47:47.598211Z","shell.execute_reply.started":"2024-03-14T07:43:12.258056Z","shell.execute_reply":"2024-03-14T07:47:47.597180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction q_earphones","metadata":{}},{"cell_type":"code","source":"df_submission['q_earphones'] = predict_any_column('q_earphones', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:47:47.600234Z","iopub.execute_input":"2024-03-14T07:47:47.600607Z","iopub.status.idle":"2024-03-14T07:50:29.415480Z","shell.execute_reply.started":"2024-03-14T07:47:47.600567Z","shell.execute_reply":"2024-03-14T07:50:29.414457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction q_thermal_preference","metadata":{}},{"cell_type":"code","source":"df_submission['q_thermal_preference'] = predict_any_column('q_thermal_preference', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:50:29.416602Z","iopub.execute_input":"2024-03-14T07:50:29.416891Z","iopub.status.idle":"2024-03-14T07:52:36.251405Z","shell.execute_reply.started":"2024-03-14T07:50:29.416859Z","shell.execute_reply":"2024-03-14T07:52:36.250388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Postprocesiing\nThe labels were predicted independently, so, some combinations of the labels might not be possible, so this sample code try to fix few of these labels","metadata":{}},{"cell_type":"code","source":"# Convert classes back form integers to strings\nfor key in dict_cat:\n    if key in df_solution.columns and key in ['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']:\n        df_submission[key] = df_submission[key].replace(dict_cat[key])\n# Select only necessary columns for submission\nsub = df_submission[['id_unique', 'q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']].set_index('id_unique')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:52:36.254028Z","iopub.execute_input":"2024-03-14T07:52:36.254328Z","iopub.status.idle":"2024-03-14T07:52:36.270641Z","shell.execute_reply.started":"2024-03-14T07:52:36.254290Z","shell.execute_reply":"2024-03-14T07:52:36.269761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Do a little bit of post processing\nfor index, row in sub.iterrows():\n    if row['q_noise_nearby'] == 'A little' and row['q_noise_kind'] == 'Not applicable' and row['q_earphones'] == 'Not applicable':\n        sub.at[index, 'q_earphones'] = 'No earphones'","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:52:36.271786Z","iopub.execute_input":"2024-03-14T07:52:36.272107Z","iopub.status.idle":"2024-03-14T07:52:36.557523Z","shell.execute_reply.started":"2024-03-14T07:52:36.272078Z","shell.execute_reply":"2024-03-14T07:52:36.556733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission file","metadata":{}},{"cell_type":"code","source":"sub.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T07:52:36.558750Z","iopub.execute_input":"2024-03-14T07:52:36.559136Z","iopub.status.idle":"2024-03-14T07:52:36.584987Z","shell.execute_reply.started":"2024-03-14T07:52:36.559099Z","shell.execute_reply":"2024-03-14T07:52:36.584285Z"},"trusted":true},"execution_count":null,"outputs":[]}]}