{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61145,"databundleVersionId":7047669,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Applications","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"This notebook has to be created through the competition","metadata":{}},{"cell_type":"markdown","source":"## Table of Contents\n* [Read data](#Read-data)\n* [Data pre-processing](#Data-pre-processing)\n* [Map of all weather stations that measure air temperature](#Map-of-all-weather-stations-that-measure-air-temperature)\n* [What's next?](#What's-next)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport math\n\nfrom datetime import datetime, timedelta\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.metrics import confusion_matrix\nfrom math import cos, asin, sqrt, pi\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\n\npd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:00:25.760216Z","iopub.execute_input":"2024-03-14T22:00:25.760610Z","iopub.status.idle":"2024-03-14T22:00:25.769640Z","shell.execute_reply.started":"2024-03-14T22:00:25.760579Z","shell.execute_reply":"2024-03-14T22:00:25.768499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read data\nReads multiple CSV files containing data related to training and testing, weather data such as rain, temperature, humidity, wind speed and direction, and weather stations.","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0],\n                       keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'],\n                       date_format='%Y-%m-%d %H:%M:%S.%f%z') #In this line, The CSV file is reading. At the same time\n                        # he consider few parameters as columns in th DataFrame: Index, without NaN data as Cadena vacia, Convert data in Date and hour's obj.\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0],\n                          keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                          date_format='%Y-%m-%d %H:%M:%S.%f%z')# Similar to the line above \ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0]) #The parse_dates and index_col parameters indicate that the first column (index) contains dates and should be interpreted as such.\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])# Similar to the line above \ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])# Similar to the line above \ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])# Similar to the line above \ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])# Similar to the line above \ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])# Similar to the line above and delete the unnamed column.","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:00:25.771708Z","iopub.execute_input":"2024-03-14T22:00:25.772044Z","iopub.status.idle":"2024-03-14T22:02:22.340359Z","shell.execute_reply.started":"2024-03-14T22:00:25.771971Z","shell.execute_reply":"2024-03-14T22:02:22.339480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.info()\n#df_stations: This is the DataFrame that contains the data about the weather stations.\n#.info(): This is the DataFrame method used to print a concise summary of the DataFrame information. Provide the following details:\n    #The total number of entries (rows) in the DataFrame.\n    #The names of the columns and their data types.\n    #The number of non-null values ​​present in each column.\n    #The amount of memory used by the DataFrame.","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:22.341673Z","iopub.execute_input":"2024-03-14T22:02:22.342134Z","iopub.status.idle":"2024-03-14T22:02:22.354747Z","shell.execute_reply.started":"2024-03-14T22:02:22.342094Z","shell.execute_reply":"2024-03-14T22:02:22.353796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data pre-processing\n\n### Check dates\n\nWhen checking the last date of different kinds of dataframes it becomes clear that the watch survey responses only last until Jan 21, 2023, while the weather data lasts until July 3, 2023.","metadata":{}},{"cell_type":"code","source":"#to print the maximum index(last date) value of the DataFrames\nprint(df_train.index.max())\nprint(df_solution.index.max())\nprint(df_rain.index.max())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:22.357828Z","iopub.execute_input":"2024-03-14T22:02:22.358248Z","iopub.status.idle":"2024-03-14T22:02:22.379071Z","shell.execute_reply.started":"2024-03-14T22:02:22.358212Z","shell.execute_reply":"2024-03-14T22:02:22.377976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't really need the weather data after Jan 21, 2023, so we might just as well get rid of it.","metadata":{}},{"cell_type":"code","source":"#These lines remove all entries from weather data DataFrames that occur after January 21, 2023.\ndf_rain = df_rain[df_rain.index<'2023-01-22']\ndf_temperature = df_temperature[df_temperature.index<'2023-01-22']\ndf_humidity = df_humidity[df_humidity.index<'2023-01-22']\ndf_wind_speed = df_wind_speed[df_wind_speed.index<'2023-01-22']\ndf_wind_direction = df_wind_direction[df_wind_direction.index<'2023-01-22']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:22.380322Z","iopub.execute_input":"2024-03-14T22:02:22.380605Z","iopub.status.idle":"2024-03-14T22:02:22.416424Z","shell.execute_reply.started":"2024-03-14T22:02:22.380580Z","shell.execute_reply":"2024-03-14T22:02:22.415397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check weather data","metadata":{}},{"cell_type":"code","source":"# Show new shortened air temperature dataframe\ndf_temperature.describe()#displays a statistical summary of the DataFrame df_temperature, which contains air temperature data.","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:22.418796Z","iopub.execute_input":"2024-03-14T22:02:22.419566Z","iopub.status.idle":"2024-03-14T22:02:22.545079Z","shell.execute_reply.started":"2024-03-14T22:02:22.419529Z","shell.execute_reply":"2024-03-14T22:02:22.544063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like there is no temperature data available in before Jan 21, 2023 for weather station S117. So, we might just as well remove that column","metadata":{}},{"cell_type":"code","source":"df_temperature = df_temperature.drop(columns=['S117'])#delete the column named 'S117' from the DataFrame\n# Get dimensions of the air temperature data\ndf_temperature.shape#the number of rows and columns","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:22.546804Z","iopub.execute_input":"2024-03-14T22:02:22.547274Z","iopub.status.idle":"2024-03-14T22:02:22.557704Z","shell.execute_reply.started":"2024-03-14T22:02:22.547237Z","shell.execute_reply":"2024-03-14T22:02:22.556666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The weather dataframes are rather large. This can make the plotting of the data slow and the output images large. To make it a bit lighter, we are going to resample the data hourly.","metadata":{"execution":{"iopub.status.busy":"2024-03-12T21:09:04.412698Z","iopub.execute_input":"2024-03-12T21:09:04.413286Z","iopub.status.idle":"2024-03-12T21:09:04.419237Z","shell.execute_reply.started":"2024-03-12T21:09:04.413253Z","shell.execute_reply":"2024-03-12T21:09:04.41811Z"}}},{"cell_type":"code","source":"# Resamples weather data DataFrames to average the data over 60-minute intervals.\ndf_temperature_average = df_temperature.resample('60min').mean()\ndf_rain_sum = df_rain.resample('60min').sum() # note: for rain the data is aggregated with a sum instead of the mean\ndf_humidity_average = df_humidity.resample('60min').mean()\ndf_wind_speed_average = df_wind_speed.resample('60min').mean()\ndf_wind_direction_average = df_wind_direction.resample('60min').mean()\n\n# Check size of resulting dataframe\ndf_temperature_average.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:22.559393Z","iopub.execute_input":"2024-03-14T22:02:22.559900Z","iopub.status.idle":"2024-03-14T22:02:22.642007Z","shell.execute_reply.started":"2024-03-14T22:02:22.559861Z","shell.execute_reply":"2024-03-14T22:02:22.641031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Map watch survey questions to question ID\n","metadata":{}},{"cell_type":"code","source":"#defines a dictionary called ws_questions that maps keys (short questions) to their respective full descriptions.\nws_questions = {\"q_noise_nearby\": \"Noise distractions nearby? (without earphones)\",\n                \"q_noise_kind\": \"What kind of noise?\",\n                \"q_earphones\": \"Wearing earphones?\",\n                \"q_thermal_preference\": \"Thermally, what do you prefer now?\",\n                \"q_location\": \"Where are you?\",\n                \"q_location_office\": \"What kind of office?\",\n                \"q_location_transport\": \"What kind of transport?\",\n                \"q_alone_group\": \"Alone or in a group?\",\n                \"q_activity_category_alone\": \"Category of activity? (alone)\",\n                \"q_activity_category_group\": \"Category of activity? (group)\"}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:22.643401Z","iopub.execute_input":"2024-03-14T22:02:22.643708Z","iopub.status.idle":"2024-03-14T22:02:22.649286Z","shell.execute_reply.started":"2024-03-14T22:02:22.643681Z","shell.execute_reply":"2024-03-14T22:02:22.648295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot weather data\n\n### Plot weather weather data time-series¶\n\nBelow we are going to plot the air temperature data from all weather stations. I used the Matplotlib method plt.plot() instead of sns.lineplot(). The reason is that plt.plot() shows the gaps in the data, while sns.lineplot() draws straight lines accross the gaps.\n","metadata":{}},{"cell_type":"code","source":"\n#This code creates a series of line graphs to display time series of air temperature data from all available weather stations.\ncols_num = 1\nrows_num = len(df_temperature_average.columns)\ncol_current = 0\nrow_current = 0 \nfig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\nplt.subplots_adjust(hspace = 0.5)\n\nfor i, col in enumerate(df_temperature_average.columns):\n    row_current = i\n    axs[row_current].plot(df_temperature_average.index, df_temperature_average[col])\n    station_name = df_stations[df_stations['id']==col]['name'].values[0]\n    axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n    axs[row_current].set_ylabel('Air Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:22.653207Z","iopub.execute_input":"2024-03-14T22:02:22.653502Z","iopub.status.idle":"2024-03-14T22:02:28.391944Z","shell.execute_reply.started":"2024-03-14T22:02:22.653471Z","shell.execute_reply":"2024-03-14T22:02:28.390690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since, we are going to plot more weather data, we might just as well wrap the above code for plotting in to a function.","metadata":{}},{"cell_type":"code","source":"# Function that plots weather data fromm all weather stations\n#This function encapsulates the code for plotting weather data into a reusable function that takes a DataFrame of weather data and a label for the Y axis as input, and returns the shape and axes of the resulting graph.\ndef plot_weather(df, ylabel):\n    cols_num = 1\n    rows_num = len(df.columns)\n    col_current = 0\n    row_current = 0 \n    fig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\n    plt.subplots_adjust(hspace = 0.5)\n\n    for i, col in enumerate(df.columns):\n        row_current = i\n        axs[row_current].plot(df.index, df[col])\n        station_name = df_stations[df_stations['id']==col]['name'].values[0]\n        axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n        axs[row_current].set_ylabel(ylabel)\n    \n    return fig, axs","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:28.393715Z","iopub.execute_input":"2024-03-14T22:02:28.394515Z","iopub.status.idle":"2024-03-14T22:02:28.402525Z","shell.execute_reply.started":"2024-03-14T22:02:28.394466Z","shell.execute_reply":"2024-03-14T22:02:28.401441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Temperature","metadata":{}},{"cell_type":"code","source":"#creates a time series graph of air temperature averaged over 60 minute intervals for all available weather stations\nfig, axs = plot_weather(df_temperature_average, 'Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:28.403818Z","iopub.execute_input":"2024-03-14T22:02:28.404248Z","iopub.status.idle":"2024-03-14T22:02:34.039384Z","shell.execute_reply.started":"2024-03-14T22:02:28.404221Z","shell.execute_reply":"2024-03-14T22:02:34.036529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Rainfall","metadata":{}},{"cell_type":"code","source":"#creates a time series graph of the total amount of precipitation (rain) accumulated in 60-minute intervals for all available weather stations.\nfig, axs = plot_weather(df_rain_sum, 'Rainfall [mm/h]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:34.040803Z","iopub.execute_input":"2024-03-14T22:02:34.041168Z","iopub.status.idle":"2024-03-14T22:02:57.431115Z","shell.execute_reply.started":"2024-03-14T22:02:34.041136Z","shell.execute_reply":"2024-03-14T22:02:57.429455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relative humidity","metadata":{}},{"cell_type":"code","source":"#creates a time series graph of average relative humidity in 60-minute intervals for all available weather stations.\nfig, axs = plot_weather(df_humidity_average, 'Relative humidity [%]')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:02:57.433049Z","iopub.execute_input":"2024-03-14T22:02:57.433891Z","iopub.status.idle":"2024-03-14T22:03:03.015399Z","shell.execute_reply.started":"2024-03-14T22:02:57.433854Z","shell.execute_reply":"2024-03-14T22:03:03.014416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creates a time series graph of average wind direction in 60-minute intervals for all available weather stations.\nfig, axs = plot_weather(df_wind_direction_average, 'Wind direction [°]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:03:03.016645Z","iopub.execute_input":"2024-03-14T22:03:03.017201Z","iopub.status.idle":"2024-03-14T22:03:09.044602Z","shell.execute_reply.started":"2024-03-14T22:03:03.017167Z","shell.execute_reply":"2024-03-14T22:03:09.043519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Map of all weather stations that measure air temperature","metadata":{}},{"cell_type":"code","source":"# Have a look at the weather station meta data\ndf_stations.head() #They represent the geographical coordinates of the meteorological stations. You can verify this by running","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:03:09.046009Z","iopub.execute_input":"2024-03-14T22:03:09.046539Z","iopub.status.idle":"2024-03-14T22:03:09.058343Z","shell.execute_reply.started":"2024-03-14T22:03:09.046490Z","shell.execute_reply":"2024-03-14T22:03:09.057327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The first value represents the number of rows in the DataFrame, that is, the number of weather stations.\nThe second value represents the number of columns in the DataFrame, that is, the number of attributes or characteristics that are recorded for each weather station.","metadata":{}},{"cell_type":"code","source":"df_stations.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:03:09.059752Z","iopub.execute_input":"2024-03-14T22:03:09.060080Z","iopub.status.idle":"2024-03-14T22:03:09.069751Z","shell.execute_reply.started":"2024-03-14T22:03:09.060054Z","shell.execute_reply":"2024-03-14T22:03:09.068844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a look at the air temperature data\ndf_temperature.columns#each column represents a specific weather station\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:03:09.070853Z","iopub.execute_input":"2024-03-14T22:03:09.071142Z","iopub.status.idle":"2024-03-14T22:03:09.080765Z","shell.execute_reply.started":"2024-03-14T22:03:09.071118Z","shell.execute_reply":"2024-03-14T22:03:09.079883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, the list of all weather stations is much longer then the list with weather stations that can measure air temperature. Hence, we need to pre-process the data a bit and only pick the weather stations that can measure temperature data.","metadata":{}},{"cell_type":"code","source":"# Initialize a new dataframe for creating the map\ndf_map = pd.DataFrame(columns=['latitude', 'longitude', 'text'])#to create a map showing the locations of weather stations that can measure air temperature\n\n\n# Go through each column of the temperature dataframe\nfor col in df_temperature.columns:\n    # Get the weather station name based on the station id\n    station_name = df_stations.loc[df_stations.id==col].name.values[0]\n    \n    # Create a new row, with latitude, longitude, hover text, and marker color as columns\n    df_row = pd.DataFrame([{'latitude': df_stations.loc[df_stations.id==col].latitude.values[0], \n                            'longitude': df_stations.loc[df_stations.id==col].longitude.values[0], \n                            'text': f'Station ID: {col} <br>Station name: {station_name} <br>Measurement: Rain',\n                            'color': 'rgb(0, 128, 255)'}])\n    \n    # Append new row to map dataframe\n    df_map = pd.concat([df_map, df_row])\n    \ndf_map.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:03:09.081861Z","iopub.execute_input":"2024-03-14T22:03:09.082143Z","iopub.status.idle":"2024-03-14T22:03:09.145673Z","shell.execute_reply.started":"2024-03-14T22:03:09.082119Z","shell.execute_reply":"2024-03-14T22:03:09.144759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pre-processing for the maps is done. Now, we can go and actually create the map chart.","metadata":{}},{"cell_type":"markdown","source":"This code creates an interactive map using the Plotly library, with markers representing the locations of weather stations that can measure air temperature. Each marker contains information about the station ID and station name. The user can interact with the map, zoom and move within it to explore the locations of weather stations.\n","metadata":{}},{"cell_type":"code","source":"# Create trace and add markers to map\ntrace = go.Scattermapbox(lat=df_map.latitude, \n                         lon=df_map.longitude,\n                         mode='markers',\n                         marker=dict(size=10,\n                                     color=df_map['color'],\n                                     opacity=0.8),\n                         hovertemplate = \"\",\n                         text=df_map['text'])\n\n# Define the layout for the map\nlayout = go.Layout(\n    margin={\"l\": 0, \"r\": 0, \"t\": 0, \"b\": 0},\n    mapbox=dict(\n        style='carto-positron', # Alternative: 'open-street-map',\n        center=dict(lat=df_map.latitude.median(),\n                    lon=df_map.longitude.median()),\n        zoom=10,\n        ),\n    height=700, width=900)\n\n# Create figure\nfig = go.Figure(data=[trace], layout=layout)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:13:09.723397Z","iopub.status.idle":"2024-03-14T22:13:09.723762Z","shell.execute_reply.started":"2024-03-14T22:13:09.723588Z","shell.execute_reply":"2024-03-14T22:13:09.723604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, you can hovering the cursor over the markers will reveal some meta information about the weather station.","metadata":{}},{"cell_type":"markdown","source":"# Plot heart rate of all participants\nAgain Matplotlib was chosen over Seaborn because it takes much less time to create the charts. Further, this time only the markers are plotted without connecting lines. This way only real data is shown and gaps in the data are preserved. Due to the irregular logging interval of the health data is much more difficult to determine when to draw a line between two markers and when to omit it.","metadata":{}},{"cell_type":"markdown","source":"1. Creating a list of unique participants:\nThe unique identifiers of the participants are extracted from the training DataFrame (df_train) using the unique() method, and then the identifiers are sorted. This creates a list of list_participants that contains the unique identifiers of all participants in the data set.\n\n2. Chart Layout Settings:\nThe number of rows and columns necessary to accommodate all participants in the chart layout is calculated. Next, a figure and a set of axes are created using plt.subplots(), with the size of the figure adjusted by the number of rows.\n\n3. Loop over participants:\nIt iterates over the list of participants (list_participants). For each participant, the data in the training DataFrame is filtered to obtain only the rows corresponding to that participant. Additionally, rows where heart rate (ts_heart_rate) is not available (NaN) are removed. Next, a scatter plot is created using scatter, where the points represent the participant's heart rate as a function of time (index). Set the chart title to \"Participant ID: {participant_id}\" and the Y axis to \"Heart Rate [bpm]\".\n\n4. Adjusting subgraph spacing:\nAdjust the vertical spacing between subplots using plt.subplots_adjust().","metadata":{}},{"cell_type":"code","source":"#generates a series of scatter plots to visualize the heart rate of each participant in the training data set.\n\nlist_participants = df_train.id_participant.unique()\nlist_participants.sort()\ncols_num = 1\nrows_num = len(list_participants)\ncol_current = 0\nrow_current = 0 \nfig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\nplt.subplots_adjust(hspace = 0.5)\n\nfor i, id_participant in enumerate(list_participants):\n    row_current = i\n    df_participant = df_train[df_train.id_participant == id_participant]\n    df_participant = df_participant[df_participant['ts_heart_rate'].notna()]\n    axs[row_current].scatter(df_participant.index, df_participant['ts_heart_rate'], marker='.')\n    axs[row_current].set_title(f'Participant ID: {id_participant}')\n    axs[row_current].set_ylabel('Heart Rate [bpm]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:03:09.573155Z","iopub.execute_input":"2024-03-14T22:03:09.573424Z","iopub.status.idle":"2024-03-14T22:04:14.262518Z","shell.execute_reply.started":"2024-03-14T22:03:09.573400Z","shell.execute_reply":"2024-03-14T22:04:14.261494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot all health kit data for one participant\nThis code generates a series of graphs to display various health measurements for a specific participant in the training data set.\n","metadata":{}},{"cell_type":"markdown","source":"1. Definition of health columns:\nA list called list_health_cols is created that contains the names of columns that represent various health measurements, such as environmental audio exposure, heart rate, oxygen saturation, and so on.\n\n2. Participant selection and data preparation:\nA specific participant, identified by id_participant_current, is selected from the training DataFrame (df_train). The index of the resulting DataFrame (df_participant) is then reset to facilitate manipulation of the data in the subsequent loop.\n\n3. Chart Layout Settings:\nThe number of rows and columns necessary to accommodate all health measurements in the chart layout is calculated. Next, a figure and a set of axes are created using plt.subplots(), with the size of the figure adjusted by the number of rows.\n\n4. Loop over health columns:\nIt iterates over the list of health columns (list_health_cols). For each column, a line is drawn using plot, where the points represent the values ​​of the health measure over time. Set the chart title to \"Participant ID: {id_participant_current}\" and the Y axis to the name of the health column.\n\n5. Adjusting subgraph spacing:\nAdjust the vertical spacing between subplots using plt.subplots_adjust().","metadata":{}},{"cell_type":"code","source":"list_health_cols = ['ts_audio_exposure_environment', \n                    'ts_heart_rate',\n                    'ts_oxygen_saturation', \n                    'ts_resting_heart_rate', \n                    'ts_stand_time',\n                    'ts_step_count', \n                    'ts_walking_distance']\n\nid_participant_current = 'xesh001'\ndf_participant = df_train[df_train.id_participant == id_participant_current]\ndf_participant = df_participant.reset_index()\ncols_num = 1\nrows_num = len(list_health_cols)\ncol_current = 0\nrow_current = 0 \nfig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\nplt.subplots_adjust(hspace = 0.5)\n\nfor i, col in enumerate(list_health_cols):\n    row_current = i\n    axs[row_current].plot(df_participant['time'], df_participant[col], marker='.', linestyle = 'None')\n    axs[row_current].set_title(f'Participant ID: {id_participant_current}')\n    axs[row_current].set_ylabel(col)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:04:14.264261Z","iopub.execute_input":"2024-03-14T22:04:14.264819Z","iopub.status.idle":"2024-03-14T22:04:16.838583Z","shell.execute_reply.started":"2024-03-14T22:04:14.264783Z","shell.execute_reply":"2024-03-14T22:04:16.837586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0], keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:13:24.272915Z","iopub.execute_input":"2024-03-14T22:13:24.273301Z","iopub.status.idle":"2024-03-14T22:15:21.253364Z","shell.execute_reply.started":"2024-03-14T22:13:24.273273Z","shell.execute_reply":"2024-03-14T22:15:21.252494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering\n\nCreate dataframes for the wind direction sine and cosine","metadata":{}},{"cell_type":"code","source":"df_wind_direction_sin = df_wind_direction.copy(deep=True)\ndf_wind_direction_cos = df_wind_direction.copy(deep=True)\nfor s in df_wind_direction.columns:\n    df_wind_direction_sin[s] = np.sin(np.pi/180*df_wind_direction_sin[s])\n    df_wind_direction_cos[s] = np.cos(np.pi/180*df_wind_direction_cos[s])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:20:50.169122Z","iopub.execute_input":"2024-03-14T22:20:50.169805Z","iopub.status.idle":"2024-03-14T22:20:50.378419Z","shell.execute_reply.started":"2024-03-14T22:20:50.169770Z","shell.execute_reply":"2024-03-14T22:20:50.377560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['source']='train'\ndf_solution['source']='solution'\ndf_all = pd.concat([df_train, df_solution])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:20:57.919024Z","iopub.execute_input":"2024-03-14T22:20:57.919828Z","iopub.status.idle":"2024-03-14T22:20:58.350149Z","shell.execute_reply.started":"2024-03-14T22:20:57.919796Z","shell.execute_reply":"2024-03-14T22:20:58.349325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['q_Indoor'] = df_all['q_location'].str.contains(\"Indoor\")\ndf_all['q_Class'] = df_all['q_location'].isin(['Indoor - Class'])\ndf_all['q_Home'] = df_all['q_location'].isin(['Indoor - Home'])\ndf_all['q_Office'] = df_all['q_location'].isin(['Indoor - Office'])\ndf_all['q_Indoor_Other'] = df_all['q_location'].isin(['Indoor - Other'])\ndf_all['q_Outdoor'] = df_all['q_location'].isin(['Outdoor'])\ndf_all['q_Transportation'] = df_all['q_location'].isin(['Transportation'])\ndf_all['q_alone_group'] = df_all['q_alone_group'].isin(['Alone'])\ndf_all['q_Bus'] = df_all['q_location_transport'].isin(['Bus'])\ndf_all['q_Car'] = df_all['q_location_transport'].isin(['Car'])\ndf_all['q_Taxi'] = df_all['q_location_transport'].isin(['Taxi'])\ndf_all['q_Train'] = df_all['q_location_transport'].isin(['Train'])\ndf_all['q_Transport_Other'] = df_all['q_location_transport'].isin(['Other'])\ndf_all['q_Focus'] = df_all['q_activity_category_alone'].isin(['Focus'])\ndf_all['q_Leisure'] = df_all['q_activity_category_alone'].isin(['Leisure'])\ndf_all['q_Alone_Activity_Other'] = df_all['q_activity_category_alone'].isin(['Other'])\ndf_all['q_Collaborate'] = df_all['q_activity_category_group'].isin(['Collaborate'])\ndf_all['q_Learn'] = df_all['q_activity_category_group'].isin(['Learn'])\ndf_all['q_Socialize'] = df_all['q_activity_category_group'].isin(['Socialize'])\ndf_all['q_Group_Activity_Other'] = df_all['q_activity_category_group'].isin(['Other'])\ndf_all['q_Conference room'] = df_all['q_location_office'].isin(['Conference room'])\ndf_all['q_Cubicles'] = df_all['q_location_office'].isin(['Cubicles'])\ndf_all['q_Individual'] = df_all['q_location_office'].isin(['Individual'])\ndf_all['q_Large open plan'] = df_all['q_location_office'].isin(['Large open plan'])\ndf_all['q_Not applicable'] = df_all['q_location_office'].isin(['Not applicable'])\ndf_all['q_Small shared'] = df_all['q_location_office'].isin(['Small shared'])\n\ndf_all = df_all.drop(columns=['q_location','q_location_transport','q_activity_category_alone','q_activity_category_group','q_location_office'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:21:03.161648Z","iopub.execute_input":"2024-03-14T22:21:03.162042Z","iopub.status.idle":"2024-03-14T22:21:05.118536Z","shell.execute_reply.started":"2024-03-14T22:21:03.162009Z","shell.execute_reply":"2024-03-14T22:21:05.117702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['duration'] = np.log((df_all.index - df_all['ws_timestamp_start']).dt.total_seconds())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:21:11.965680Z","iopub.execute_input":"2024-03-14T22:21:11.966097Z","iopub.status.idle":"2024-03-14T22:21:12.018897Z","shell.execute_reply.started":"2024-03-14T22:21:11.966064Z","shell.execute_reply":"2024-03-14T22:21:12.017848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['time_of_day_cos'] = np.cos(np.deg2rad((df_all.index.hour*60 + df_all.index.minute)/4))\ndf_all['time_of_day_sin'] = np.sin(np.deg2rad((df_all.index.hour*60 + df_all.index.minute)/4))\ndf_all['time_of_day'] = (df_all.index.hour*60 + df_all.index.minute)\ndf_all['day_of_week'] = df_all.index.dayofweek\n#df_all = df_all.reset_index()\n#df_all = df_all.sort_values(['id_participant', 'time'])\n#df_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:21:18.272977Z","iopub.execute_input":"2024-03-14T22:21:18.273415Z","iopub.status.idle":"2024-03-14T22:21:18.877618Z","shell.execute_reply.started":"2024-03-14T22:21:18.273382Z","shell.execute_reply":"2024-03-14T22:21:18.876831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['med_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('median')\ndf_all['med_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('median')\ndf_all['med_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('median')\ndf_all['med_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('median')\ndf_all['med_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('median')\ndf_all['med_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('median')\ndf_all['med_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('median')\n\ndf_all['mean_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('mean')\ndf_all['mean_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('mean')\ndf_all['mean_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('mean')\ndf_all['mean_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('mean')\ndf_all['mean_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('mean')\ndf_all['mean_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('mean')\ndf_all['mean_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('mean')\n\ndf_all['std_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('std')\ndf_all['std_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('std')\ndf_all['std_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('std')\ndf_all['std_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('std')\ndf_all['std_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('std')\ndf_all['std_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('std')\ndf_all['std_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('std')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:21:25.760878Z","iopub.execute_input":"2024-03-14T22:21:25.761281Z","iopub.status.idle":"2024-03-14T22:21:29.752366Z","shell.execute_reply.started":"2024-03-14T22:21:25.761250Z","shell.execute_reply":"2024-03-14T22:21:29.751522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['ts_resting_heart_rate'] = (df_all['ts_resting_heart_rate'] - df_all['mean_resting_heart_rate'])/df_all['std_resting_heart_rate']\ndf_all['ts_audio_exposure_environment'] = (df_all['ts_audio_exposure_environment'] - df_all['mean_audio_exposure_environment'])/df_all['std_audio_exposure_environment']\ndf_all['ts_oxygen_saturation'] = (df_all['ts_oxygen_saturation'] - df_all['mean_oxygen_saturation'])/df_all['std_oxygen_saturation']\ndf_all['ts_stand_time'] = (df_all['ts_stand_time'] - df_all['mean_stand_time'])/df_all['std_stand_time']\ndf_all['ts_heart_rate'] = (df_all['ts_heart_rate'] - df_all['mean_heart_rate'])/df_all['std_heart_rate']\ndf_all['ts_step_count'] = (df_all['ts_step_count'] - df_all['mean_step_count'])/df_all['std_step_count']\ndf_all['ts_walking_distance'] = (df_all['ts_walking_distance'] - df_all['mean_walking_distance'])/df_all['std_walking_distance']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:21:35.927795Z","iopub.execute_input":"2024-03-14T22:21:35.928470Z","iopub.status.idle":"2024-03-14T22:21:36.023689Z","shell.execute_reply.started":"2024-03-14T22:21:35.928439Z","shell.execute_reply":"2024-03-14T22:21:36.022916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Interpolate the features provided by Apple HealthKit\nHealth data, e.g., ts_heart_reate, and the watch survey responses, e.g., q_thermal_preference, do not have coinciding timestamps. Hence, the health data needs to be interpolated at the timestamps of the watch survey responses.\n\nThe interpolated values are stored in columns with the prefix ws_ which stands for 'watch survey', e.g., the interpolated values of ts_heart_rate are stored in ws_heart_rate.","metadata":{}},{"cell_type":"code","source":"# Function to resample health data at the timestamps of watch survey responses.\ndef resample_health(df, col_name, col_name_new):\n    df[col_name_new] = df[col_name].interpolate(method='linear').bfill().ffill().mask(df['ws_survey_count'].isna())\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:21:47.182611Z","iopub.execute_input":"2024-03-14T22:21:47.182957Z","iopub.status.idle":"2024-03-14T22:21:47.188280Z","shell.execute_reply.started":"2024-03-14T22:21:47.182930Z","shell.execute_reply":"2024-03-14T22:21:47.187253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resample normalized health data\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_heart_rate', col_name_new='ws_heart_rate').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_audio_exposure_environment', col_name_new='ws_audio_exposure_environment').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_oxygen_saturation', col_name_new='ws_oxygen_saturation').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_resting_heart_rate', col_name_new='ws_resting_heart_rate').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_stand_time', col_name_new='ws_stand_time').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_step_count', col_name_new='ws_step_count').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_walking_distance', col_name_new='ws_walking_distance').reset_index(level=[0], drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:21:56.235578Z","iopub.execute_input":"2024-03-14T22:21:56.236307Z","iopub.status.idle":"2024-03-14T22:22:28.806688Z","shell.execute_reply.started":"2024-03-14T22:21:56.236275Z","shell.execute_reply":"2024-03-14T22:22:28.805812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Interpolate weather data\nThe weather data is distributed over several dataframes. Each measurement, e.g., rain or air temperature, is in a separate dataframe. The column names are the IDs of the weather stations. There is one dataframe (df_stations), which contains meta information to the weather stations, e.g., coordinates and names.\n\nTo make the weather data usable, we need to find the nearest weather stations for each watch survey response in df_all. Then we can transfer the measurement values with the same timestamp to from the weather data dataframe, e.g., df_rain, to the dataframe with the Cozie data (df_all).","metadata":{}},{"cell_type":"code","source":"# Function to get weather data for a given timestamp and location from the nearest weather station\ndef get_weather_data(df_weather_data, lat, lon, timestamp):    \n    # Get timestamp with no seconds or microseconds\n    timestamp = timestamp.replace(second=0, microsecond=0)\n    \n    # Get weather data for timestamp above\n    weather_data_row = df_weather_data[(df_weather_data.index >= timestamp- timedelta(hours=0, minutes=10)) & (df_weather_data.index <= timestamp + timedelta(hours=0, minutes=10))]\n\n    # If no weather data is available for timestamp above, get weather data for next minute\n    if (weather_data_row.empty):\n        timestamp = timestamp + pd.Timedelta(minutes=1)\n        weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n        # If there is still no weather data available, return NaN\n        if (weather_data_row.empty):\n            timestamp = timestamp - 2 * pd.Timedelta(minutes=1)\n            weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n            if (weather_data_row.empty):\n                return np.NaN\n\n    # Get weather station ids from weather stations with with data\n    station_ids_with_data = weather_data_row.columns[weather_data_row.count() > 0].tolist()\n\n    # Get weather stations with data\n    df_stations2 = df_stations[df_stations.id.isin(station_ids_with_data)]\n\n    # Compute distance to each weather station\n    df_stations2['distance'] = df_stations2.apply(lambda x: distance(x['latitude'], x['longitude'], lat, lon), axis=1)\n\n    # Get nearest weather station\n    station_id_nearest = df_stations2[df_stations2.distance==df_stations2.distance.min()].id.values[0]\n\n    # Retrun NaN for cases where the nearest weather station is more than 10km away\n    if df_stations2.distance.min()>10:\n        return np.NaN\n\n    # Return weather data from nearest weather station\n    return np.nanmean(weather_data_row[station_id_nearest])\n\n# Function to compute distnace on spherical earth\ndef distance(lat1, lon1, lat2, lon2):\n    r = 6371 # km\n    p = pi / 180\n\n    a = 0.5 - cos((lat2-lat1)*p)/2 + cos(lat1*p) * cos(lat2*p) * (1-cos((lon2-lon1)*p))/2\n    return 2 * r * asin(sqrt(a))","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:22:52.867924Z","iopub.execute_input":"2024-03-14T22:22:52.868322Z","iopub.status.idle":"2024-03-14T22:22:52.882259Z","shell.execute_reply.started":"2024-03-14T22:22:52.868292Z","shell.execute_reply":"2024-03-14T22:22:52.881235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply above function to the dataset for all weather data\ndf_all = df_all.reset_index()\ndf_all['ws_rainfall'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_rain, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_air_temperature'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_temperature, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_relative_humidity'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_humidity, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_speed'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_speed, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_sin'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_sin, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_cos'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_cos, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:23:00.182816Z","iopub.execute_input":"2024-03-14T22:23:00.183841Z","iopub.status.idle":"2024-03-14T22:27:55.208200Z","shell.execute_reply.started":"2024-03-14T22:23:00.183796Z","shell.execute_reply":"2024-03-14T22:27:55.207144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Remove unused rows and columns ","metadata":{}},{"cell_type":"code","source":"# Remove all rows that are not a watch surve response\ndf_all = df_all[df_all.ws_survey_count.notna()]\n\n# Remove some timestamp columns\ndf_all = df_all.drop(columns=['ws_timestamp_location', 'ws_timestamp_start'])\n\n# Remove columns of the original health data those are no longer needed since we are going to use the interpolated values\ndf_all = df_all.drop(columns=['ts_heart_rate',\n                              'ts_audio_exposure_environment',\n                              'ts_oxygen_saturation',\n                              'ts_resting_heart_rate',\n                              'ts_stand_time', \n                              'ts_step_count',\n                              'ts_walking_distance'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:28:36.907106Z","iopub.execute_input":"2024-03-14T22:28:36.907509Z","iopub.status.idle":"2024-03-14T22:28:36.987420Z","shell.execute_reply.started":"2024-03-14T22:28:36.907479Z","shell.execute_reply":"2024-03-14T22:28:36.986315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering (Part 2)\nThe computation of these features is faster after we have dropped lots of survery rows","metadata":{}},{"cell_type":"code","source":"earphone_dict = df_all.groupby(['id_participant','q_earphones'])['duration'].count().to_dict()\ndf_all['own_earphones'] = 0\ndf_all['own_noise_cancelling'] = 0\nfor index, row in df_all.iterrows():\n    if (row['id_participant'], 'Earphones') in earphone_dict:\n        df_all.at[index, 'own_earphones'] = 1\n    if (row['id_participant'], 'Noise cancelling') in earphone_dict:\n        df_all.at[index, 'own_noise_cancelling'] = 1","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:28:46.510901Z","iopub.execute_input":"2024-03-14T22:28:46.511282Z","iopub.status.idle":"2024-03-14T22:28:48.650336Z","shell.execute_reply.started":"2024-03-14T22:28:46.511253Z","shell.execute_reply":"2024-03-14T22:28:48.649252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Use_earphone_in_class = set(df_all[(df_all['q_Class']) & (df_all['q_earphones']=='Earphones')]['id_participant'].unique())\nbeen_to_class = set(df_all[(df_all['q_Class'])&(df_all['source'] == 'train')]['id_participant'].unique())\ndiff = been_to_class.difference(Use_earphone_in_class)\nfor index, row in df_all.iterrows():\n    if row['q_Class']:\n        df_all.at[index, 'own_noise_cancelling'] = 0\n        if row['id_participant'] in diff:\n            df_all.at[index, 'own_earphones'] = 0","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:28:54.290973Z","iopub.execute_input":"2024-03-14T22:28:54.291709Z","iopub.status.idle":"2024-03-14T22:28:55.069751Z","shell.execute_reply.started":"2024-03-14T22:28:54.291673Z","shell.execute_reply":"2024-03-14T22:28:55.068921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['location_count'] = 0\nfor index, row in df_all.iterrows():\n    df_all.at[index, 'location_count'] = len(df_all[(df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:29:03.206067Z","iopub.execute_input":"2024-03-14T22:29:03.206433Z","iopub.status.idle":"2024-03-14T22:29:14.898915Z","shell.execute_reply.started":"2024-03-14T22:29:03.206405Z","shell.execute_reply":"2024-03-14T22:29:14.898051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['location_prob'] = 0\nfor index, row in df_all.iterrows():\n    temp = df_all[(df_all['id_participant']==row['id_participant']) & (df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])]\n    prob = len(temp) / len(df_all[(df_all['id_participant']==row['id_participant'])])\n    df_all.at[index, 'location_prob'] = prob","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:29:21.926155Z","iopub.execute_input":"2024-03-14T22:29:21.926903Z","iopub.status.idle":"2024-03-14T22:30:15.809063Z","shell.execute_reply.started":"2024-03-14T22:29:21.926868Z","shell.execute_reply":"2024-03-14T22:30:15.808203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split data\nWe split the data into the training set and solution set according to the original data","metadata":{}},{"cell_type":"code","source":"# Separte training and solution data again\ndf_train = df_all[df_all.source == 'train'].copy()\ndf_solution = df_all[df_all.source == 'solution'].copy()\n\n# Drop the helper columns ws_surve_count_2 and source again\ndf_train = df_train.drop(columns=['source'])\ndf_solution = df_solution.drop(columns=['source'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:30:21.706069Z","iopub.execute_input":"2024-03-14T22:30:21.706411Z","iopub.status.idle":"2024-03-14T22:30:21.810780Z","shell.execute_reply.started":"2024-03-14T22:30:21.706386Z","shell.execute_reply":"2024-03-14T22:30:21.809901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preparation\nFor the machine learning model below, all input and output values need to be numeric. At this point, the watch survey the response columns, e.g., q_noise_nearby, are of type string. Hence, we need convert the values in these columns into numbers.","metadata":{}},{"cell_type":"code","source":"# Find columns with values that are not numeric\ncat_vars = list(df_train.select_dtypes(include=['object']).columns)\n# encoding categorical variables\ndict_cat = {}\n\nfor var in cat_vars:\n    if var == 'id_participant': # no need to encode the participant ID\n        continue\n    # convert categorical variables into 'category' type\n    df_train[var] = df_train[var].astype('category')\n    df_solution[var] = df_solution[var].astype('category')\n    \n    # dictionary to keep track of the variable mapping: {variable-name: {number1: string1, number2: string2, ...}\n    dict_cat[var] = dict(enumerate(df_train[var].cat.categories))\n\n    # encode variables, missing values are encoded as -1\n    df_train[var] = df_train[var].cat.codes\n    df_solution[var] = df_solution[var].cat.codes","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:30:30.815941Z","iopub.execute_input":"2024-03-14T22:30:30.816334Z","iopub.status.idle":"2024-03-14T22:30:30.843396Z","shell.execute_reply.started":"2024-03-14T22:30:30.816296Z","shell.execute_reply":"2024-03-14T22:30:30.842438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict any column\nWe are going to wrap up above code into a function in order to make the prediction of the other columns a bit more convenient. The steps are mostly the same as above and might feel a bit repetitive.","metadata":{}},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier\nfrom sklearn.ensemble import VotingClassifier, StackingClassifier","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:30:37.089722Z","iopub.execute_input":"2024-03-14T22:30:37.090206Z","iopub.status.idle":"2024-03-14T22:30:37.095040Z","shell.execute_reply.started":"2024-03-14T22:30:37.090174Z","shell.execute_reply":"2024-03-14T22:30:37.094045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_any_column(column_name, df_all, df_solution):\n    # Remove rows where the predicting variable is missing\n    df_all = df_all[df_all[column_name] != -1]\n    \n    # Remove rows where the predicting variable is missing\n    df_all = df_all.drop(columns=['id_participant'])\n    df_solution = df_solution.drop(columns=['id_participant'])\n    \n    # Count classes in predicting variable\n    num_class = df_train[column_name].unique()\n    \n    # Break dataframe into variables and ground truth\n    y_all = df_all[column_name].to_numpy()\n    X_all = df_all.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)#.to_numpy()\n    cat_features = [f for f in cat_vars if f in X_all.columns]\n    X_all2 = X_all.to_numpy()\n    X_solution = df_solution.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)\n    X_solution2 = X_solution.to_numpy()\n    # Select features\n    #sel = SelectFromModel(XGBClassifier(num_class=num_class, objective='multi:softmax', enable_categorical=False))\n    #sel.fit(X_all,y_all)\n    #selected_feat= X_all.columns[(sel.get_support())]\n    arr = np.zeros((len(num_class),len(num_class)))\n    score = 0\n    print(num_class)\n    y_solution = np.zeros(shape=(X_solution.shape[0],len(num_class)))\n    y_train = np.zeros(shape=(X_all.shape[0],len(num_class)))\n    skf = StratifiedKFold(n_splits=5,random_state=1990,shuffle=True)\n    for i, (train_index, test_index) in enumerate(skf.split(X_all, y_all)):\n        # Initialize model\n        models = [          \n           ('catBoost',make_pipeline(CatBoostClassifier(iterations=1000, verbose=1000, random_seed=42,learning_rate=0.05))),          \n           ('lgbm',make_pipeline(lgb.LGBMClassifier(random_state=42))),   \n           ('XGB',make_pipeline(XGBClassifier(objective='multi:softmax',random_state=42)))     \n            \n        ]\n        voter = VotingClassifier(models, voting = 'soft')\n        # Predict and evaluate\n        voter.fit(X_all.iloc[train_index], y_all[train_index])\n        y_pred = voter.predict_proba(X_all.iloc[test_index]) \n        \n        score += accuracy_score(y_all[test_index], np.argmax(y_pred, axis=1))/5\n        arr = arr + confusion_matrix(y_all[test_index], np.argmax(y_pred, axis=1))\n\n        # Predict solution\n        y_solution += voter.predict_proba(X_solution) #+ clf2.predict_proba(X_solution2)\n    \n    print('Model accuracy score: {0:0.4f}'. format(score))\n    #print(arr)\n    \n    return np.argmax(y_solution, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:30:42.928407Z","iopub.execute_input":"2024-03-14T22:30:42.928758Z","iopub.status.idle":"2024-03-14T22:30:42.942481Z","shell.execute_reply.started":"2024-03-14T22:30:42.928729Z","shell.execute_reply":"2024-03-14T22:30:42.941525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a copy of the solution dataframe for submission\ndf_submission = df_solution.copy()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:30:50.800014Z","iopub.execute_input":"2024-03-14T22:30:50.800728Z","iopub.status.idle":"2024-03-14T22:30:50.836303Z","shell.execute_reply.started":"2024-03-14T22:30:50.800695Z","shell.execute_reply":"2024-03-14T22:30:50.834782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_noise_nearby'] = predict_any_column('q_noise_nearby', df_train, df_solution)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:30:55.815096Z","iopub.execute_input":"2024-03-14T22:30:55.815772Z","iopub.status.idle":"2024-03-14T22:32:58.813769Z","shell.execute_reply.started":"2024-03-14T22:30:55.815738Z","shell.execute_reply":"2024-03-14T22:32:58.812670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_noise_kind'] = predict_any_column('q_noise_kind', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:33:13.856375Z","iopub.execute_input":"2024-03-14T22:33:13.856749Z","iopub.status.idle":"2024-03-14T22:37:40.668678Z","shell.execute_reply.started":"2024-03-14T22:33:13.856718Z","shell.execute_reply":"2024-03-14T22:37:40.667666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_earphones'] = predict_any_column('q_earphones', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:38:01.308089Z","iopub.execute_input":"2024-03-14T22:38:01.308470Z","iopub.status.idle":"2024-03-14T22:40:38.191771Z","shell.execute_reply.started":"2024-03-14T22:38:01.308442Z","shell.execute_reply":"2024-03-14T22:40:38.190607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_thermal_preference'] = predict_any_column('q_thermal_preference', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:41:42.224295Z","iopub.execute_input":"2024-03-14T22:41:42.224707Z","iopub.status.idle":"2024-03-14T22:43:44.808421Z","shell.execute_reply.started":"2024-03-14T22:41:42.224675Z","shell.execute_reply":"2024-03-14T22:43:44.807429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Perform some postprocesiing\nThe labels were predicted independently, so, some combinations of the labels might not be possible, so this sample code try to fix few of these labels","metadata":{}},{"cell_type":"code","source":"# Convert classes back form integers to strings\nfor key in dict_cat:\n    if key in df_solution.columns and key in ['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']:\n        df_submission[key] = df_submission[key].replace(dict_cat[key])\n# Select only necessary columns for submission\nsub = df_submission[['id_unique', 'q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']].set_index('id_unique')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:43:58.567778Z","iopub.execute_input":"2024-03-14T22:43:58.568450Z","iopub.status.idle":"2024-03-14T22:43:58.585118Z","shell.execute_reply.started":"2024-03-14T22:43:58.568415Z","shell.execute_reply":"2024-03-14T22:43:58.584035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Do a little bit of post processing\nfor index, row in sub.iterrows():\n    if row['q_noise_nearby'] == 'A little' and row['q_noise_kind'] == 'Not applicable' and row['q_earphones'] == 'Not applicable':\n        sub.at[index, 'q_earphones'] = 'No earphones'","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:44:09.955905Z","iopub.execute_input":"2024-03-14T22:44:09.956309Z","iopub.status.idle":"2024-03-14T22:44:10.237053Z","shell.execute_reply.started":"2024-03-14T22:44:09.956278Z","shell.execute_reply":"2024-03-14T22:44:10.236226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create submission file","metadata":{}},{"cell_type":"code","source":"# Write dataframe to csv file\nsub.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T22:44:19.939505Z","iopub.execute_input":"2024-03-14T22:44:19.939854Z","iopub.status.idle":"2024-03-14T22:44:19.964276Z","shell.execute_reply.started":"2024-03-14T22:44:19.939828Z","shell.execute_reply":"2024-03-14T22:44:19.963289Z"},"trusted":true},"execution_count":null,"outputs":[]}]}