{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61145,"databundleVersionId":7047669,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Applications","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"This notebook has to be created through the competition","metadata":{}},{"cell_type":"markdown","source":"## Table of Contents\n* [Read data](#Read-data)\n* [Data pre-processing](#Data-pre-processing)\n* [Map of all weather stations that measure air temperature](#Map-of-all-weather-stations-that-measure-air-temperature)\n* [What's next?](#What's-next)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport math","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:40:31.705361Z","iopub.execute_input":"2024-03-14T20:40:31.706096Z","iopub.status.idle":"2024-03-14T20:40:31.710709Z","shell.execute_reply.started":"2024-03-14T20:40:31.706068Z","shell.execute_reply":"2024-03-14T20:40:31.709768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], \n                       keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                       date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0],\n                          keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                          date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:40:31.712144Z","iopub.execute_input":"2024-03-14T20:40:31.712450Z","iopub.status.idle":"2024-03-14T20:42:25.407831Z","shell.execute_reply.started":"2024-03-14T20:40:31.712421Z","shell.execute_reply":"2024-03-14T20:42:25.406639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:25.409623Z","iopub.execute_input":"2024-03-14T20:42:25.409982Z","iopub.status.idle":"2024-03-14T20:42:25.419771Z","shell.execute_reply.started":"2024-03-14T20:42:25.409950Z","shell.execute_reply":"2024-03-14T20:42:25.418874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data pre-processing\n\n### Check dates\n\nWhen checking the last date of different kinds of dataframes it becomes clear that the watch survey responses only last until Jan 21, 2023, while the weather data lasts until July 3, 2023.","metadata":{}},{"cell_type":"code","source":"print(df_train.index.max())\nprint(df_solution.index.max())\nprint(df_rain.index.max())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:25.420996Z","iopub.execute_input":"2024-03-14T20:42:25.421310Z","iopub.status.idle":"2024-03-14T20:42:25.440783Z","shell.execute_reply.started":"2024-03-14T20:42:25.421267Z","shell.execute_reply":"2024-03-14T20:42:25.439900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't really need the weather data after Jan 21, 2023, so we might just as well get rid of it.","metadata":{}},{"cell_type":"code","source":"df_rain = df_rain[df_rain.index<'2023-01-22']\ndf_temperature = df_temperature[df_temperature.index<'2023-01-22']\ndf_humidity = df_humidity[df_humidity.index<'2023-01-22']\ndf_wind_speed = df_wind_speed[df_wind_speed.index<'2023-01-22']\ndf_wind_direction = df_wind_direction[df_wind_direction.index<'2023-01-22']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:25.442818Z","iopub.execute_input":"2024-03-14T20:42:25.443071Z","iopub.status.idle":"2024-03-14T20:42:25.475994Z","shell.execute_reply.started":"2024-03-14T20:42:25.443049Z","shell.execute_reply":"2024-03-14T20:42:25.475291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check weather data","metadata":{}},{"cell_type":"code","source":"# Show new shortened air temperature dataframe\ndf_temperature.describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:25.476927Z","iopub.execute_input":"2024-03-14T20:42:25.477152Z","iopub.status.idle":"2024-03-14T20:42:25.593682Z","shell.execute_reply.started":"2024-03-14T20:42:25.477131Z","shell.execute_reply":"2024-03-14T20:42:25.592898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like there is no temperature data available in before Jan 21, 2023 for weather station S117. So, we might just as well remove that column","metadata":{}},{"cell_type":"code","source":"df_temperature = df_temperature.drop(columns=['S117'])\n# Get dimensions of the air temperature data\ndf_temperature.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:25.594880Z","iopub.execute_input":"2024-03-14T20:42:25.595145Z","iopub.status.idle":"2024-03-14T20:42:25.603780Z","shell.execute_reply.started":"2024-03-14T20:42:25.595122Z","shell.execute_reply":"2024-03-14T20:42:25.602782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The weather dataframes are rather large. This can make the plotting of the data slow and the output images large. To make it a bit lighter, we are going to resample the data hourly.","metadata":{"execution":{"iopub.status.busy":"2024-03-12T21:09:04.412698Z","iopub.execute_input":"2024-03-12T21:09:04.413286Z","iopub.status.idle":"2024-03-12T21:09:04.419237Z","shell.execute_reply.started":"2024-03-12T21:09:04.413253Z","shell.execute_reply":"2024-03-12T21:09:04.41811Z"}}},{"cell_type":"code","source":"# Resample weather data\ndf_temperature_average = df_temperature.resample('60min').mean()\ndf_rain_sum = df_rain.resample('60min').sum() # note: for rain the data is aggregated with a sum instead of the mean\ndf_humidity_average = df_humidity.resample('60min').mean()\ndf_wind_speed_average = df_wind_speed.resample('60min').mean()\ndf_wind_direction_average = df_wind_direction.resample('60min').mean()\n\n# Check size of resulting dataframe\ndf_temperature_average.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:25.605044Z","iopub.execute_input":"2024-03-14T20:42:25.605426Z","iopub.status.idle":"2024-03-14T20:42:25.683655Z","shell.execute_reply.started":"2024-03-14T20:42:25.605391Z","shell.execute_reply":"2024-03-14T20:42:25.682793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Map watch survey questions to question ID\n","metadata":{}},{"cell_type":"code","source":"ws_questions = {\"q_noise_nearby\": \"Noise distractions nearby? (without earphones)\",\n                \"q_noise_kind\": \"What kind of noise?\",\n                \"q_earphones\": \"Wearing earphones?\",\n                \"q_thermal_preference\": \"Thermally, what do you prefer now?\",\n                \"q_location\": \"Where are you?\",\n                \"q_location_office\": \"What kind of office?\",\n                \"q_location_transport\": \"What kind of transport?\",\n                \"q_alone_group\": \"Alone or in a group?\",\n                \"q_activity_category_alone\": \"Category of activity? (alone)\",\n                \"q_activity_category_group\": \"Category of activity? (group)\"}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:25.684860Z","iopub.execute_input":"2024-03-14T20:42:25.685118Z","iopub.status.idle":"2024-03-14T20:42:25.690128Z","shell.execute_reply.started":"2024-03-14T20:42:25.685095Z","shell.execute_reply":"2024-03-14T20:42:25.689260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot weather data\n\n### Plot weather weather data time-series¶\n\nBelow we are going to plot the air temperature data from all weather stations. I used the Matplotlib method plt.plot() instead of sns.lineplot(). The reason is that plt.plot() shows the gaps in the data, while sns.lineplot() draws straight lines accross the gaps.\n","metadata":{}},{"cell_type":"code","source":"cols_num = 1\nrows_num = len(df_temperature_average.columns)\ncol_current = 0\nrow_current = 0 \nfig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\nplt.subplots_adjust(hspace = 0.5)\n\nfor i, col in enumerate(df_temperature_average.columns):\n    row_current = i\n    axs[row_current].plot(df_temperature_average.index, df_temperature_average[col])\n    station_name = df_stations[df_stations['id']==col]['name'].values[0]\n    axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n    axs[row_current].set_ylabel('Air Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:25.691346Z","iopub.execute_input":"2024-03-14T20:42:25.691634Z","iopub.status.idle":"2024-03-14T20:42:30.966492Z","shell.execute_reply.started":"2024-03-14T20:42:25.691604Z","shell.execute_reply":"2024-03-14T20:42:30.965247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since, we are going to plot more weather data, we might just as well wrap the above code for plotting in to a function.","metadata":{}},{"cell_type":"code","source":"# Function that plots weather data fromm all weather stations\ndef plot_weather(df, ylabel):\n    cols_num = 1\n    rows_num = len(df.columns)\n    col_current = 0\n    row_current = 0 \n    fig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\n    plt.subplots_adjust(hspace = 0.5)\n\n    for i, col in enumerate(df.columns):\n        row_current = i\n        axs[row_current].plot(df.index, df[col])\n        station_name = df_stations[df_stations['id']==col]['name'].values[0]\n        axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n        axs[row_current].set_ylabel(ylabel)\n    \n    return fig, axs","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:30.969979Z","iopub.execute_input":"2024-03-14T20:42:30.970267Z","iopub.status.idle":"2024-03-14T20:42:30.977181Z","shell.execute_reply.started":"2024-03-14T20:42:30.970243Z","shell.execute_reply":"2024-03-14T20:42:30.976091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Temperature","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_temperature_average, 'Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:30.978376Z","iopub.execute_input":"2024-03-14T20:42:30.978655Z","iopub.status.idle":"2024-03-14T20:42:36.479613Z","shell.execute_reply.started":"2024-03-14T20:42:30.978633Z","shell.execute_reply":"2024-03-14T20:42:36.478364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Rainfall","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_rain_sum, 'Rainfall [mm/h]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:36.480960Z","iopub.execute_input":"2024-03-14T20:42:36.481250Z","iopub.status.idle":"2024-03-14T20:42:58.837693Z","shell.execute_reply.started":"2024-03-14T20:42:36.481225Z","shell.execute_reply":"2024-03-14T20:42:58.833400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relative humidity","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_humidity_average, 'Relative humidity [%]')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:42:58.838954Z","iopub.execute_input":"2024-03-14T20:42:58.839241Z","iopub.status.idle":"2024-03-14T20:43:04.120162Z","shell.execute_reply.started":"2024-03-14T20:42:58.839215Z","shell.execute_reply":"2024-03-14T20:43:04.118653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Map of all weather stations that measure air temperature","metadata":{}},{"cell_type":"code","source":"# Have a look at the weather station meta data\ndf_stations.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:43:04.121389Z","iopub.execute_input":"2024-03-14T20:43:04.121746Z","iopub.status.idle":"2024-03-14T20:43:04.137140Z","shell.execute_reply.started":"2024-03-14T20:43:04.121712Z","shell.execute_reply":"2024-03-14T20:43:04.135483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:43:04.138711Z","iopub.execute_input":"2024-03-14T20:43:04.139003Z","iopub.status.idle":"2024-03-14T20:43:04.151481Z","shell.execute_reply.started":"2024-03-14T20:43:04.138978Z","shell.execute_reply":"2024-03-14T20:43:04.148735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a look at the air temperature data\ndf_temperature.columns","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:43:04.153054Z","iopub.execute_input":"2024-03-14T20:43:04.153355Z","iopub.status.idle":"2024-03-14T20:43:04.161693Z","shell.execute_reply.started":"2024-03-14T20:43:04.153328Z","shell.execute_reply":"2024-03-14T20:43:04.160128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, the list of all weather stations is much longer then the list with weather stations that can measure air temperature. Hence, we need to pre-process the data a bit and only pick the weather stations that can measure temperature data.","metadata":{}},{"cell_type":"code","source":"# Initialize a new dataframe for creating the map\ndf_map = pd.DataFrame(columns=['latitude', 'longitude', 'text'])\n\n\n# Go through each column of the temperature dataframe\nfor col in df_temperature.columns:\n    # Get the weather station name based on the station id\n    station_name = df_stations.loc[df_stations.id==col].name.values[0]\n    \n    # Create a new row, with latitude, longitude, hover text, and marker color as columns\n    df_row = pd.DataFrame([{'latitude': df_stations.loc[df_stations.id==col].latitude.values[0], \n                            'longitude': df_stations.loc[df_stations.id==col].longitude.values[0], \n                            'text': f'Station ID: {col} <br>Station name: {station_name} <br>Measurement: Rain',\n                            'color': 'rgb(0, 128, 255)'}])\n    \n    # Append new row to map dataframe\n    df_map = pd.concat([df_map, df_row])\n    \ndf_map.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:43:04.163255Z","iopub.execute_input":"2024-03-14T20:43:04.163553Z","iopub.status.idle":"2024-03-14T20:43:04.229284Z","shell.execute_reply.started":"2024-03-14T20:43:04.163527Z","shell.execute_reply":"2024-03-14T20:43:04.228159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pre-processing for the maps is done. Now, we can go and actually create the map chart.","metadata":{}},{"cell_type":"code","source":"# Create trace and add markers to map\ntrace = go.Scattermapbox(lat=df_map.latitude, \n                         lon=df_map.longitude,\n                         mode='markers',\n                         marker=dict(size=10,\n                                     color=df_map['color'],\n                                     opacity=0.8),\n                         hovertemplate = \"\",\n                         text=df_map['text'])\n\n# Define the layout for the map\nlayout = go.Layout(\n    margin={\"l\": 0, \"r\": 0, \"t\": 0, \"b\": 0},\n    mapbox=dict(\n        style='carto-positron', # Alternative: 'open-street-map',\n        center=dict(lat=df_map.latitude.median(),\n                    lon=df_map.longitude.median()),\n        zoom=10,\n        ),\n    height=700, width=900)\n\n# Create figure\nfig = go.Figure(data=[trace], layout=layout)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:43:04.230424Z","iopub.execute_input":"2024-03-14T20:43:04.230671Z","iopub.status.idle":"2024-03-14T20:43:04.249603Z","shell.execute_reply.started":"2024-03-14T20:43:04.230649Z","shell.execute_reply":"2024-03-14T20:43:04.248582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, you can hovering the cursor over the markers will reveal some meta information about the weather station.","metadata":{}},{"cell_type":"markdown","source":"## What's next?","metadata":{}},{"cell_type":"markdown","source":"How do I continue this? Where to start?!\n\nGO TO THE COMPETITION SITE!\n\n[https://www.kaggle.com/competitions/cool-quiet-city-competition](https://www.kaggle.com/competitions/cool-quiet-city-competition)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SOLUTION FROM 1ST PLACE","metadata":{}},{"cell_type":"markdown","source":"### Packages","metadata":{}},{"cell_type":"code","source":"\nimport pandas as pd \nimport numpy as np\nfrom datetime import datetime, timedelta\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.metrics import confusion_matrix\nfrom math import cos, asin, sqrt, pi\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\n\npd.set_option('display.max_columns', None)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:43:04.250750Z","iopub.execute_input":"2024-03-14T20:43:04.251057Z","iopub.status.idle":"2024-03-14T20:43:09.713832Z","shell.execute_reply.started":"2024-03-14T20:43:04.251032Z","shell.execute_reply":"2024-03-14T20:43:09.712844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading CSV Files","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0], keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:43:09.715365Z","iopub.execute_input":"2024-03-14T20:43:09.715670Z","iopub.status.idle":"2024-03-14T20:45:04.197902Z","shell.execute_reply.started":"2024-03-14T20:43:09.715644Z","shell.execute_reply":"2024-03-14T20:45:04.196682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ingeniería","metadata":{}},{"cell_type":"code","source":"df_wind_direction_sin = df_wind_direction.copy(deep=True)\ndf_wind_direction_cos = df_wind_direction.copy(deep=True)\nfor s in df_wind_direction.columns:\n    df_wind_direction_sin[s] = np.sin(np.pi/180*df_wind_direction_sin[s])\n    df_wind_direction_cos[s] = np.cos(np.pi/180*df_wind_direction_cos[s])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:04.199114Z","iopub.execute_input":"2024-03-14T20:45:04.199450Z","iopub.status.idle":"2024-03-14T20:45:04.406146Z","shell.execute_reply.started":"2024-03-14T20:45:04.199421Z","shell.execute_reply":"2024-03-14T20:45:04.405159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['source']='train'\ndf_solution['source']='solution'\ndf_all = pd.concat([df_train, df_solution])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:04.407335Z","iopub.execute_input":"2024-03-14T20:45:04.407626Z","iopub.status.idle":"2024-03-14T20:45:04.911902Z","shell.execute_reply.started":"2024-03-14T20:45:04.407602Z","shell.execute_reply":"2024-03-14T20:45:04.911131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['q_Indoor'] = df_all['q_location'].str.contains(\"Indoor\")\ndf_all['q_Class'] = df_all['q_location'].isin(['Indoor - Class'])\ndf_all['q_Home'] = df_all['q_location'].isin(['Indoor - Home'])\ndf_all['q_Office'] = df_all['q_location'].isin(['Indoor - Office'])\ndf_all['q_Indoor_Other'] = df_all['q_location'].isin(['Indoor - Other'])\ndf_all['q_Outdoor'] = df_all['q_location'].isin(['Outdoor'])\ndf_all['q_Transportation'] = df_all['q_location'].isin(['Transportation'])\ndf_all['q_alone_group'] = df_all['q_alone_group'].isin(['Alone'])\ndf_all['q_Bus'] = df_all['q_location_transport'].isin(['Bus'])\ndf_all['q_Car'] = df_all['q_location_transport'].isin(['Car'])\ndf_all['q_Taxi'] = df_all['q_location_transport'].isin(['Taxi'])\ndf_all['q_Train'] = df_all['q_location_transport'].isin(['Train'])\ndf_all['q_Transport_Other'] = df_all['q_location_transport'].isin(['Other'])\ndf_all['q_Focus'] = df_all['q_activity_category_alone'].isin(['Focus'])\ndf_all['q_Leisure'] = df_all['q_activity_category_alone'].isin(['Leisure'])\ndf_all['q_Alone_Activity_Other'] = df_all['q_activity_category_alone'].isin(['Other'])\ndf_all['q_Collaborate'] = df_all['q_activity_category_group'].isin(['Collaborate'])\ndf_all['q_Learn'] = df_all['q_activity_category_group'].isin(['Learn'])\ndf_all['q_Socialize'] = df_all['q_activity_category_group'].isin(['Socialize'])\ndf_all['q_Group_Activity_Other'] = df_all['q_activity_category_group'].isin(['Other'])\ndf_all['q_Conference room'] = df_all['q_location_office'].isin(['Conference room'])\ndf_all['q_Cubicles'] = df_all['q_location_office'].isin(['Cubicles'])\ndf_all['q_Individual'] = df_all['q_location_office'].isin(['Individual'])\ndf_all['q_Large open plan'] = df_all['q_location_office'].isin(['Large open plan'])\ndf_all['q_Not applicable'] = df_all['q_location_office'].isin(['Not applicable'])\ndf_all['q_Small shared'] = df_all['q_location_office'].isin(['Small shared'])\n\ndf_all = df_all.drop(columns=['q_location','q_location_transport','q_activity_category_alone','q_activity_category_group','q_location_office'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:04.912923Z","iopub.execute_input":"2024-03-14T20:45:04.913174Z","iopub.status.idle":"2024-03-14T20:45:06.944047Z","shell.execute_reply.started":"2024-03-14T20:45:04.913151Z","shell.execute_reply":"2024-03-14T20:45:06.943275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['duration'] = np.log((df_all.index - df_all['ws_timestamp_start']).dt.total_seconds())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:06.945317Z","iopub.execute_input":"2024-03-14T20:45:06.946070Z","iopub.status.idle":"2024-03-14T20:45:07.001742Z","shell.execute_reply.started":"2024-03-14T20:45:06.946037Z","shell.execute_reply":"2024-03-14T20:45:07.000994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['time_of_day_cos'] = np.cos(np.deg2rad((df_all.index.hour*60 + df_all.index.minute)/4))\ndf_all['time_of_day_sin'] = np.sin(np.deg2rad((df_all.index.hour*60 + df_all.index.minute)/4))\ndf_all['time_of_day'] = (df_all.index.hour*60 + df_all.index.minute)\ndf_all['day_of_week'] = df_all.index.dayofweek\n#df_all = df_all.reset_index()\n#df_all = df_all.sort_values(['id_participant', 'time'])\n#df_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:07.002768Z","iopub.execute_input":"2024-03-14T20:45:07.003016Z","iopub.status.idle":"2024-03-14T20:45:07.586225Z","shell.execute_reply.started":"2024-03-14T20:45:07.002994Z","shell.execute_reply":"2024-03-14T20:45:07.585464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['med_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('median')\ndf_all['med_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('median')\ndf_all['med_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('median')\ndf_all['med_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('median')\ndf_all['med_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('median')\ndf_all['med_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('median')\ndf_all['med_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('median')\n\ndf_all['mean_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('mean')\ndf_all['mean_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('mean')\ndf_all['mean_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('mean')\ndf_all['mean_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('mean')\ndf_all['mean_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('mean')\ndf_all['mean_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('mean')\ndf_all['mean_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('mean')\n\ndf_all['std_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('std')\ndf_all['std_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('std')\ndf_all['std_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('std')\ndf_all['std_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('std')\ndf_all['std_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('std')\ndf_all['std_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('std')\ndf_all['std_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('std')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:07.587417Z","iopub.execute_input":"2024-03-14T20:45:07.587694Z","iopub.status.idle":"2024-03-14T20:45:11.963391Z","shell.execute_reply.started":"2024-03-14T20:45:07.587670Z","shell.execute_reply":"2024-03-14T20:45:11.962422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['ts_resting_heart_rate'] = (df_all['ts_resting_heart_rate'] - df_all['mean_resting_heart_rate'])/df_all['std_resting_heart_rate']\ndf_all['ts_audio_exposure_environment'] = (df_all['ts_audio_exposure_environment'] - df_all['mean_audio_exposure_environment'])/df_all['std_audio_exposure_environment']\ndf_all['ts_oxygen_saturation'] = (df_all['ts_oxygen_saturation'] - df_all['mean_oxygen_saturation'])/df_all['std_oxygen_saturation']\ndf_all['ts_stand_time'] = (df_all['ts_stand_time'] - df_all['mean_stand_time'])/df_all['std_stand_time']\ndf_all['ts_heart_rate'] = (df_all['ts_heart_rate'] - df_all['mean_heart_rate'])/df_all['std_heart_rate']\ndf_all['ts_step_count'] = (df_all['ts_step_count'] - df_all['mean_step_count'])/df_all['std_step_count']\ndf_all['ts_walking_distance'] = (df_all['ts_walking_distance'] - df_all['mean_walking_distance'])/df_all['std_walking_distance']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:11.964537Z","iopub.execute_input":"2024-03-14T20:45:11.964814Z","iopub.status.idle":"2024-03-14T20:45:12.060168Z","shell.execute_reply.started":"2024-03-14T20:45:11.964790Z","shell.execute_reply":"2024-03-14T20:45:12.059432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Interpolando","metadata":{}},{"cell_type":"code","source":"# Function to resample health data at the timestamps of watch survey responses.\ndef resample_health(df, col_name, col_name_new):\n    df[col_name_new] = df[col_name].interpolate(method='linear').bfill().ffill().mask(df['ws_survey_count'].isna())\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:12.066611Z","iopub.execute_input":"2024-03-14T20:45:12.067258Z","iopub.status.idle":"2024-03-14T20:45:12.071774Z","shell.execute_reply.started":"2024-03-14T20:45:12.067234Z","shell.execute_reply":"2024-03-14T20:45:12.070801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resample normalized health data\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_heart_rate', col_name_new='ws_heart_rate').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_audio_exposure_environment', col_name_new='ws_audio_exposure_environment').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_oxygen_saturation', col_name_new='ws_oxygen_saturation').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_resting_heart_rate', col_name_new='ws_resting_heart_rate').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_stand_time', col_name_new='ws_stand_time').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_step_count', col_name_new='ws_step_count').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_walking_distance', col_name_new='ws_walking_distance').reset_index(level=[0], drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:12.072852Z","iopub.execute_input":"2024-03-14T20:45:12.073108Z","iopub.status.idle":"2024-03-14T20:45:46.767349Z","shell.execute_reply.started":"2024-03-14T20:45:12.073086Z","shell.execute_reply":"2024-03-14T20:45:46.766351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Interpolando Weather Data","metadata":{}},{"cell_type":"code","source":"# Function to get weather data for a given timestamp and location from the nearest weather station\ndef get_weather_data(df_weather_data, lat, lon, timestamp):    \n    # Get timestamp with no seconds or microseconds\n    timestamp = timestamp.replace(second=0, microsecond=0)\n    \n    # Get weather data for timestamp above\n    weather_data_row = df_weather_data[(df_weather_data.index >= timestamp- timedelta(hours=0, minutes=10)) & (df_weather_data.index <= timestamp + timedelta(hours=0, minutes=10))]\n\n    # If no weather data is available for timestamp above, get weather data for next minute\n    if (weather_data_row.empty):\n        timestamp = timestamp + pd.Timedelta(minutes=1)\n        weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n        # If there is still no weather data available, return NaN\n        if (weather_data_row.empty):\n            timestamp = timestamp - 2 * pd.Timedelta(minutes=1)\n            weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n            if (weather_data_row.empty):\n                return np.NaN\n\n    # Get weather station ids from weather stations with with data\n    station_ids_with_data = weather_data_row.columns[weather_data_row.count() > 0].tolist()\n\n    # Get weather stations with data\n    df_stations2 = df_stations[df_stations.id.isin(station_ids_with_data)]\n\n    # Compute distance to each weather station\n    df_stations2['distance'] = df_stations2.apply(lambda x: distance(x['latitude'], x['longitude'], lat, lon), axis=1)\n\n    # Get nearest weather station\n    station_id_nearest = df_stations2[df_stations2.distance==df_stations2.distance.min()].id.values[0]\n\n    # Retrun NaN for cases where the nearest weather station is more than 10km away\n    if df_stations2.distance.min()>10:\n        return np.NaN\n\n    # Return weather data from nearest weather station\n    return np.nanmean(weather_data_row[station_id_nearest])\n\n# Function to compute distnace on spherical earth\ndef distance(lat1, lon1, lat2, lon2):\n    r = 6371 # km\n    p = pi / 180\n\n    a = 0.5 - cos((lat2-lat1)*p)/2 + cos(lat1*p) * cos(lat2*p) * (1-cos((lon2-lon1)*p))/2\n    return 2 * r * asin(sqrt(a))","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:46.768629Z","iopub.execute_input":"2024-03-14T20:45:46.769361Z","iopub.status.idle":"2024-03-14T20:45:46.781059Z","shell.execute_reply.started":"2024-03-14T20:45:46.769327Z","shell.execute_reply":"2024-03-14T20:45:46.780169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply above function to the dataset for all weather data\ndf_all = df_all.reset_index()\ndf_all['ws_rainfall'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_rain, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_air_temperature'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_temperature, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_relative_humidity'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_humidity, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_speed'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_speed, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_sin'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_sin, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_cos'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_cos, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:45:46.782234Z","iopub.execute_input":"2024-03-14T20:45:46.782524Z","iopub.status.idle":"2024-03-14T20:50:25.084900Z","shell.execute_reply.started":"2024-03-14T20:45:46.782502Z","shell.execute_reply":"2024-03-14T20:50:25.084078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Remover columnas y filas inusuales","metadata":{}},{"cell_type":"code","source":"# Remove all rows that are not a watch surve response\ndf_all = df_all[df_all.ws_survey_count.notna()]\n\n# Remove some timestamp columns\ndf_all = df_all.drop(columns=['ws_timestamp_location', 'ws_timestamp_start'])\n\n# Remove columns of the original health data those are no longer needed since we are going to use the interpolated values\ndf_all = df_all.drop(columns=['ts_heart_rate',\n                              'ts_audio_exposure_environment',\n                              'ts_oxygen_saturation',\n                              'ts_resting_heart_rate',\n                              'ts_stand_time', \n                              'ts_step_count',\n                              'ts_walking_distance'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:50:25.086215Z","iopub.execute_input":"2024-03-14T20:50:25.087017Z","iopub.status.idle":"2024-03-14T20:50:25.177592Z","shell.execute_reply.started":"2024-03-14T20:50:25.086982Z","shell.execute_reply":"2024-03-14T20:50:25.176822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ingeniería 2","metadata":{}},{"cell_type":"code","source":"earphone_dict = df_all.groupby(['id_participant','q_earphones'])['duration'].count().to_dict()\ndf_all['own_earphones'] = 0\ndf_all['own_noise_cancelling'] = 0\nfor index, row in df_all.iterrows():\n    if (row['id_participant'], 'Earphones') in earphone_dict:\n        df_all.at[index, 'own_earphones'] = 1\n    if (row['id_participant'], 'Noise cancelling') in earphone_dict:\n        df_all.at[index, 'own_noise_cancelling'] = 1","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:50:25.178753Z","iopub.execute_input":"2024-03-14T20:50:25.179082Z","iopub.status.idle":"2024-03-14T20:50:27.225412Z","shell.execute_reply.started":"2024-03-14T20:50:25.179051Z","shell.execute_reply":"2024-03-14T20:50:27.224645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Use_earphone_in_class = set(df_all[(df_all['q_Class']) & (df_all['q_earphones']=='Earphones')]['id_participant'].unique())\nbeen_to_class = set(df_all[(df_all['q_Class'])&(df_all['source'] == 'train')]['id_participant'].unique())\ndiff = been_to_class.difference(Use_earphone_in_class)\nfor index, row in df_all.iterrows():\n    if row['q_Class']:\n        df_all.at[index, 'own_noise_cancelling'] = 0\n        if row['id_participant'] in diff:\n            df_all.at[index, 'own_earphones'] = 0","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:50:27.226473Z","iopub.execute_input":"2024-03-14T20:50:27.226737Z","iopub.status.idle":"2024-03-14T20:50:27.940224Z","shell.execute_reply.started":"2024-03-14T20:50:27.226714Z","shell.execute_reply":"2024-03-14T20:50:27.939474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['location_count'] = 0\nfor index, row in df_all.iterrows():\n    df_all.at[index, 'location_count'] = len(df_all[(df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:50:27.941323Z","iopub.execute_input":"2024-03-14T20:50:27.941604Z","iopub.status.idle":"2024-03-14T20:50:38.949752Z","shell.execute_reply.started":"2024-03-14T20:50:27.941580Z","shell.execute_reply":"2024-03-14T20:50:38.948835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['location_prob'] = 0\nfor index, row in df_all.iterrows():\n    temp = df_all[(df_all['id_participant']==row['id_participant']) & (df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])]\n    prob = len(temp) / len(df_all[(df_all['id_participant']==row['id_participant'])])\n    df_all.at[index, 'location_prob'] = prob","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:50:38.951020Z","iopub.execute_input":"2024-03-14T20:50:38.951408Z","iopub.status.idle":"2024-03-14T20:51:30.402866Z","shell.execute_reply.started":"2024-03-14T20:50:38.951365Z","shell.execute_reply":"2024-03-14T20:51:30.401844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Partir la data","metadata":{}},{"cell_type":"code","source":"# Separte training and solution data again\ndf_train = df_all[df_all.source == 'train'].copy()\ndf_solution = df_all[df_all.source == 'solution'].copy()\n\n# Drop the helper columns ws_surve_count_2 and source again\ndf_train = df_train.drop(columns=['source'])\ndf_solution = df_solution.drop(columns=['source'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:51:30.404241Z","iopub.execute_input":"2024-03-14T20:51:30.404583Z","iopub.status.idle":"2024-03-14T20:51:30.493073Z","shell.execute_reply.started":"2024-03-14T20:51:30.404556Z","shell.execute_reply":"2024-03-14T20:51:30.492342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preparar la Data","metadata":{}},{"cell_type":"code","source":"# Find columns with values that are not numeric\ncat_vars = list(df_train.select_dtypes(include=['object']).columns)\n# encoding categorical variables\ndict_cat = {}\n\nfor var in cat_vars:\n    if var == 'id_participant': # no need to encode the participant ID\n        continue\n    # convert categorical variables into 'category' type\n    df_train[var] = df_train[var].astype('category')\n    df_solution[var] = df_solution[var].astype('category')\n    \n    # dictionary to keep track of the variable mapping: {variable-name: {number1: string1, number2: string2, ...}\n    dict_cat[var] = dict(enumerate(df_train[var].cat.categories))\n\n    # encode variables, missing values are encoded as -1\n    df_train[var] = df_train[var].cat.codes\n    df_solution[var] = df_solution[var].cat.codes","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:51:30.494303Z","iopub.execute_input":"2024-03-14T20:51:30.494699Z","iopub.status.idle":"2024-03-14T20:51:30.520484Z","shell.execute_reply.started":"2024-03-14T20:51:30.494664Z","shell.execute_reply":"2024-03-14T20:51:30.519604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predecir cualquier columna","metadata":{}},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier\nfrom sklearn.ensemble import VotingClassifier, StackingClassifier","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:51:30.521648Z","iopub.execute_input":"2024-03-14T20:51:30.521934Z","iopub.status.idle":"2024-03-14T20:51:30.621754Z","shell.execute_reply.started":"2024-03-14T20:51:30.521904Z","shell.execute_reply":"2024-03-14T20:51:30.620724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_any_column(column_name, df_all, df_solution):\n    # Remove rows where the predicting variable is missing\n    df_all = df_all[df_all[column_name] != -1]\n    \n    # Remove rows where the predicting variable is missing\n    df_all = df_all.drop(columns=['id_participant'])\n    df_solution = df_solution.drop(columns=['id_participant'])\n    \n    # Count classes in predicting variable\n    num_class = df_train[column_name].unique()\n    \n    # Break dataframe into variables and ground truth\n    y_all = df_all[column_name].to_numpy()\n    X_all = df_all.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)#.to_numpy()\n    cat_features = [f for f in cat_vars if f in X_all.columns]\n    X_all2 = X_all.to_numpy()\n    X_solution = df_solution.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)\n    X_solution2 = X_solution.to_numpy()\n    # Select features\n    #sel = SelectFromModel(XGBClassifier(num_class=num_class, objective='multi:softmax', enable_categorical=False))\n    #sel.fit(X_all,y_all)\n    #selected_feat= X_all.columns[(sel.get_support())]\n    arr = np.zeros((len(num_class),len(num_class)))\n    score = 0\n    print(num_class)\n    y_solution = np.zeros(shape=(X_solution.shape[0],len(num_class)))\n    y_train = np.zeros(shape=(X_all.shape[0],len(num_class)))\n    skf = StratifiedKFold(n_splits=5,random_state=1990,shuffle=True)\n    for i, (train_index, test_index) in enumerate(skf.split(X_all, y_all)):\n        # Initialize model\n        models = [          \n           ('catBoost',make_pipeline(CatBoostClassifier(iterations=1000, verbose=1000, random_seed=42,learning_rate=0.05))),          \n           ('lgbm',make_pipeline(lgb.LGBMClassifier(random_state=42))),   \n           ('XGB',make_pipeline(XGBClassifier(objective='multi:softmax',random_state=42)))     \n            \n        ]\n        voter = VotingClassifier(models, voting = 'soft')\n        # Predict and evaluate\n        voter.fit(X_all.iloc[train_index], y_all[train_index])\n        y_pred = voter.predict_proba(X_all.iloc[test_index]) \n        \n        score += accuracy_score(y_all[test_index], np.argmax(y_pred, axis=1))/5\n        arr = arr + confusion_matrix(y_all[test_index], np.argmax(y_pred, axis=1))\n\n        # Predict solution\n        y_solution += voter.predict_proba(X_solution) #+ clf2.predict_proba(X_solution2)\n    \n    print('Model accuracy score: {0:0.4f}'. format(score))\n    #print(arr)\n    \n    return np.argmax(y_solution, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:51:30.625629Z","iopub.execute_input":"2024-03-14T20:51:30.625956Z","iopub.status.idle":"2024-03-14T20:51:30.639829Z","shell.execute_reply.started":"2024-03-14T20:51:30.625933Z","shell.execute_reply":"2024-03-14T20:51:30.638828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a copy of the solution dataframe for submission\ndf_submission = df_solution.copy()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:51:30.640854Z","iopub.execute_input":"2024-03-14T20:51:30.641102Z","iopub.status.idle":"2024-03-14T20:51:30.654142Z","shell.execute_reply.started":"2024-03-14T20:51:30.641081Z","shell.execute_reply":"2024-03-14T20:51:30.653226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predecir q_noise_nearby","metadata":{}},{"cell_type":"code","source":"df_submission['q_noise_nearby'] = predict_any_column('q_noise_nearby', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:51:30.655483Z","iopub.execute_input":"2024-03-14T20:51:30.656058Z","iopub.status.idle":"2024-03-14T20:53:29.096871Z","shell.execute_reply.started":"2024-03-14T20:51:30.656028Z","shell.execute_reply":"2024-03-14T20:53:29.095822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predecir q_noise_kind","metadata":{}},{"cell_type":"code","source":"df_submission['q_noise_kind'] = predict_any_column('q_noise_kind', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:53:29.098006Z","iopub.execute_input":"2024-03-14T20:53:29.098276Z","iopub.status.idle":"2024-03-14T20:57:45.355376Z","shell.execute_reply.started":"2024-03-14T20:53:29.098253Z","shell.execute_reply":"2024-03-14T20:57:45.354434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predecir q_earphones","metadata":{}},{"cell_type":"code","source":"df_submission['q_earphones'] = predict_any_column('q_earphones', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:57:45.356753Z","iopub.execute_input":"2024-03-14T20:57:45.357116Z","iopub.status.idle":"2024-03-14T21:00:16.334238Z","shell.execute_reply.started":"2024-03-14T20:57:45.357084Z","shell.execute_reply":"2024-03-14T21:00:16.333313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predecir q_thermal_preference","metadata":{}},{"cell_type":"code","source":"df_submission['q_thermal_preference'] = predict_any_column('q_thermal_preference', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:00:16.335610Z","iopub.execute_input":"2024-03-14T21:00:16.335896Z","iopub.status.idle":"2024-03-14T21:02:13.241753Z","shell.execute_reply.started":"2024-03-14T21:00:16.335871Z","shell.execute_reply":"2024-03-14T21:02:13.240829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Perform some postprocessiing","metadata":{}},{"cell_type":"code","source":"# Convert classes back form integers to strings\nfor key in dict_cat:\n    if key in df_solution.columns and key in ['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']:\n        df_submission[key] = df_submission[key].replace(dict_cat[key])\n# Select only necessary columns for submission\nsub = df_submission[['id_unique', 'q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']].set_index('id_unique')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:02:13.242878Z","iopub.execute_input":"2024-03-14T21:02:13.243148Z","iopub.status.idle":"2024-03-14T21:02:13.258182Z","shell.execute_reply.started":"2024-03-14T21:02:13.243125Z","shell.execute_reply":"2024-03-14T21:02:13.257248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Do a little bit of post processing\nfor index, row in sub.iterrows():\n    if row['q_noise_nearby'] == 'A little' and row['q_noise_kind'] == 'Not applicable' and row['q_earphones'] == 'Not applicable':\n        sub.at[index, 'q_earphones'] = 'No earphones'","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:02:13.259286Z","iopub.execute_input":"2024-03-14T21:02:13.259594Z","iopub.status.idle":"2024-03-14T21:02:13.532657Z","shell.execute_reply.started":"2024-03-14T21:02:13.259569Z","shell.execute_reply":"2024-03-14T21:02:13.531698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Write dataframe to csv file\nsub.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:02:13.533845Z","iopub.execute_input":"2024-03-14T21:02:13.534128Z","iopub.status.idle":"2024-03-14T21:02:13.558957Z","shell.execute_reply.started":"2024-03-14T21:02:13.534104Z","shell.execute_reply":"2024-03-14T21:02:13.558210Z"},"trusted":true},"execution_count":null,"outputs":[]}]}