{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61145,"databundleVersionId":7047669,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Applications","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"This notebook has to be created through the competition","metadata":{}},{"cell_type":"markdown","source":"## Table of Contents\n* [Read data](#Read-data)\n* [Data pre-processing](#Data-pre-processing)\n* [Map of all weather stations that measure air temperature](#Map-of-all-weather-stations-that-measure-air-temperature)\n* [Solution](#Solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-13T14:57:35.443111Z","iopub.execute_input":"2024-03-13T14:57:35.443822Z","iopub.status.idle":"2024-03-13T14:57:35.451288Z","shell.execute_reply.started":"2024-03-13T14:57:35.443779Z","shell.execute_reply":"2024-03-13T14:57:35.449216Z"}}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport math","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:17:08.370720Z","iopub.execute_input":"2024-03-14T20:17:08.371132Z","iopub.status.idle":"2024-03-14T20:17:08.380021Z","shell.execute_reply.started":"2024-03-14T20:17:08.371101Z","shell.execute_reply":"2024-03-14T20:17:08.379132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], \n                       keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                       date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0],\n                          keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                          date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:17:08.380985Z","iopub.execute_input":"2024-03-14T20:17:08.383103Z","iopub.status.idle":"2024-03-14T20:19:04.973307Z","shell.execute_reply.started":"2024-03-14T20:17:08.383078Z","shell.execute_reply":"2024-03-14T20:19:04.972497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:04.974357Z","iopub.execute_input":"2024-03-14T20:19:04.974656Z","iopub.status.idle":"2024-03-14T20:19:04.984571Z","shell.execute_reply.started":"2024-03-14T20:19:04.974631Z","shell.execute_reply":"2024-03-14T20:19:04.983678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data pre-processing\n\n### Check dates\n\nWhen checking the last date of different kinds of dataframes it becomes clear that the watch survey responses only last until Jan 21, 2023, while the weather data lasts until July 3, 2023.","metadata":{}},{"cell_type":"code","source":"print(df_train.index.max())\nprint(df_solution.index.max())\nprint(df_rain.index.max())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:04.985835Z","iopub.execute_input":"2024-03-14T20:19:04.986126Z","iopub.status.idle":"2024-03-14T20:19:05.005842Z","shell.execute_reply.started":"2024-03-14T20:19:04.986103Z","shell.execute_reply":"2024-03-14T20:19:05.005040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't really need the weather data after Jan 21, 2023, so we might just as well get rid of it.","metadata":{}},{"cell_type":"code","source":"df_rain = df_rain[df_rain.index<'2023-01-22']\ndf_temperature = df_temperature[df_temperature.index<'2023-01-22']\ndf_humidity = df_humidity[df_humidity.index<'2023-01-22']\ndf_wind_speed = df_wind_speed[df_wind_speed.index<'2023-01-22']\ndf_wind_direction = df_wind_direction[df_wind_direction.index<'2023-01-22']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:05.006777Z","iopub.execute_input":"2024-03-14T20:19:05.007098Z","iopub.status.idle":"2024-03-14T20:19:05.048157Z","shell.execute_reply.started":"2024-03-14T20:19:05.007072Z","shell.execute_reply":"2024-03-14T20:19:05.047343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check weather data","metadata":{}},{"cell_type":"code","source":"# Show new shortened air temperature dataframe\ndf_temperature.describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:05.049163Z","iopub.execute_input":"2024-03-14T20:19:05.049422Z","iopub.status.idle":"2024-03-14T20:19:05.173061Z","shell.execute_reply.started":"2024-03-14T20:19:05.049400Z","shell.execute_reply":"2024-03-14T20:19:05.172116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like there is no temperature data available in before Jan 21, 2023 for weather station S117. So, we might just as well remove that column","metadata":{}},{"cell_type":"code","source":"df_temperature = df_temperature.drop(columns=['S117'])\n# Get dimensions of the air temperature data\ndf_temperature.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:05.174445Z","iopub.execute_input":"2024-03-14T20:19:05.175126Z","iopub.status.idle":"2024-03-14T20:19:05.184370Z","shell.execute_reply.started":"2024-03-14T20:19:05.175090Z","shell.execute_reply":"2024-03-14T20:19:05.183501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The weather dataframes are rather large. This can make the plotting of the data slow and the output images large. To make it a bit lighter, we are going to resample the data hourly.","metadata":{"execution":{"iopub.status.busy":"2024-03-12T21:09:04.412698Z","iopub.execute_input":"2024-03-12T21:09:04.413286Z","iopub.status.idle":"2024-03-12T21:09:04.419237Z","shell.execute_reply.started":"2024-03-12T21:09:04.413253Z","shell.execute_reply":"2024-03-12T21:09:04.41811Z"}}},{"cell_type":"code","source":"# Resample weather data\ndf_temperature_average = df_temperature.resample('60min').mean()\ndf_rain_sum = df_rain.resample('60min').sum() # note: for rain the data is aggregated with a sum instead of the mean\ndf_humidity_average = df_humidity.resample('60min').mean()\ndf_wind_speed_average = df_wind_speed.resample('60min').mean()\ndf_wind_direction_average = df_wind_direction.resample('60min').mean()\n\n# Check size of resulting dataframe\ndf_temperature_average.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:05.185456Z","iopub.execute_input":"2024-03-14T20:19:05.185728Z","iopub.status.idle":"2024-03-14T20:19:05.265012Z","shell.execute_reply.started":"2024-03-14T20:19:05.185697Z","shell.execute_reply":"2024-03-14T20:19:05.264054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Map watch survey questions to question ID\n","metadata":{}},{"cell_type":"code","source":"ws_questions = {\"q_noise_nearby\": \"Noise distractions nearby? (without earphones)\",\n                \"q_noise_kind\": \"What kind of noise?\",\n                \"q_earphones\": \"Wearing earphones?\",\n                \"q_thermal_preference\": \"Thermally, what do you prefer now?\",\n                \"q_location\": \"Where are you?\",\n                \"q_location_office\": \"What kind of office?\",\n                \"q_location_transport\": \"What kind of transport?\",\n                \"q_alone_group\": \"Alone or in a group?\",\n                \"q_activity_category_alone\": \"Category of activity? (alone)\",\n                \"q_activity_category_group\": \"Category of activity? (group)\"}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:05.266272Z","iopub.execute_input":"2024-03-14T20:19:05.266548Z","iopub.status.idle":"2024-03-14T20:19:05.271806Z","shell.execute_reply.started":"2024-03-14T20:19:05.266525Z","shell.execute_reply":"2024-03-14T20:19:05.270795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot weather data\n\n### Plot weather weather data time-series¶\n\nBelow we are going to plot the air temperature data from all weather stations. I used the Matplotlib method plt.plot() instead of sns.lineplot(). The reason is that plt.plot() shows the gaps in the data, while sns.lineplot() draws straight lines accross the gaps.\n","metadata":{}},{"cell_type":"code","source":"cols_num = 1\nrows_num = len(df_temperature_average.columns)\ncol_current = 0\nrow_current = 0 \nfig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\nplt.subplots_adjust(hspace = 0.5)\n\nfor i, col in enumerate(df_temperature_average.columns):\n    row_current = i\n    axs[row_current].plot(df_temperature_average.index, df_temperature_average[col])\n    station_name = df_stations[df_stations['id']==col]['name'].values[0]\n    axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n    axs[row_current].set_ylabel('Air Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:05.273171Z","iopub.execute_input":"2024-03-14T20:19:05.273592Z","iopub.status.idle":"2024-03-14T20:19:14.427378Z","shell.execute_reply.started":"2024-03-14T20:19:05.273563Z","shell.execute_reply":"2024-03-14T20:19:14.426466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since, we are going to plot more weather data, we might just as well wrap the above code for plotting in to a function.","metadata":{}},{"cell_type":"code","source":"# Function that plots weather data fromm all weather stations\ndef plot_weather(df, ylabel):\n    cols_num = 1\n    rows_num = len(df.columns)\n    col_current = 0\n    row_current = 0 \n    fig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\n    plt.subplots_adjust(hspace = 0.5)\n\n    for i, col in enumerate(df.columns):\n        row_current = i\n        axs[row_current].plot(df.index, df[col])\n        station_name = df_stations[df_stations['id']==col]['name'].values[0]\n        axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n        axs[row_current].set_ylabel(ylabel)\n    \n    return fig, axs","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:14.428751Z","iopub.execute_input":"2024-03-14T20:19:14.429090Z","iopub.status.idle":"2024-03-14T20:19:14.437655Z","shell.execute_reply.started":"2024-03-14T20:19:14.429061Z","shell.execute_reply":"2024-03-14T20:19:14.436715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Temperature","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_temperature_average, 'Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:14.439008Z","iopub.execute_input":"2024-03-14T20:19:14.439291Z","iopub.status.idle":"2024-03-14T20:19:24.059341Z","shell.execute_reply.started":"2024-03-14T20:19:14.439260Z","shell.execute_reply":"2024-03-14T20:19:24.058164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Rainfall","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_rain_sum, 'Rainfall [mm/h]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:19:24.060714Z","iopub.execute_input":"2024-03-14T20:19:24.061061Z","iopub.status.idle":"2024-03-14T20:20:04.223633Z","shell.execute_reply.started":"2024-03-14T20:19:24.061032Z","shell.execute_reply":"2024-03-14T20:20:04.221999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relative humidity","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_humidity_average, 'Relative humidity [%]')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:04.225011Z","iopub.execute_input":"2024-03-14T20:20:04.225332Z","iopub.status.idle":"2024-03-14T20:20:13.082579Z","shell.execute_reply.started":"2024-03-14T20:20:04.225306Z","shell.execute_reply":"2024-03-14T20:20:13.081614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Map of all weather stations that measure air temperature","metadata":{}},{"cell_type":"code","source":"# Have a look at the weather station meta data\ndf_stations.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:13.083955Z","iopub.execute_input":"2024-03-14T20:20:13.084362Z","iopub.status.idle":"2024-03-14T20:20:13.096943Z","shell.execute_reply.started":"2024-03-14T20:20:13.084334Z","shell.execute_reply":"2024-03-14T20:20:13.095949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:13.098095Z","iopub.execute_input":"2024-03-14T20:20:13.098389Z","iopub.status.idle":"2024-03-14T20:20:13.107679Z","shell.execute_reply.started":"2024-03-14T20:20:13.098365Z","shell.execute_reply":"2024-03-14T20:20:13.106858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a look at the air temperature data\ndf_temperature.columns","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:13.108817Z","iopub.execute_input":"2024-03-14T20:20:13.109369Z","iopub.status.idle":"2024-03-14T20:20:13.117645Z","shell.execute_reply.started":"2024-03-14T20:20:13.109323Z","shell.execute_reply":"2024-03-14T20:20:13.116716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, the list of all weather stations is much longer then the list with weather stations that can measure air temperature. Hence, we need to pre-process the data a bit and only pick the weather stations that can measure temperature data.","metadata":{}},{"cell_type":"code","source":"# Initialize a new dataframe for creating the map\ndf_map = pd.DataFrame(columns=['latitude', 'longitude', 'text'])\n\n\n# Go through each column of the temperature dataframe\nfor col in df_temperature.columns:\n    # Get the weather station name based on the station id\n    station_name = df_stations.loc[df_stations.id==col].name.values[0]\n    \n    # Create a new row, with latitude, longitude, hover text, and marker color as columns\n    df_row = pd.DataFrame([{'latitude': df_stations.loc[df_stations.id==col].latitude.values[0], \n                            'longitude': df_stations.loc[df_stations.id==col].longitude.values[0], \n                            'text': f'Station ID: {col} <br>Station name: {station_name} <br>Measurement: Rain',\n                            'color': 'rgb(0, 128, 255)'}])\n    \n    # Append new row to map dataframe\n    df_map = pd.concat([df_map, df_row])\ndf_map.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:13.118831Z","iopub.execute_input":"2024-03-14T20:20:13.119178Z","iopub.status.idle":"2024-03-14T20:20:13.171614Z","shell.execute_reply.started":"2024-03-14T20:20:13.119155Z","shell.execute_reply":"2024-03-14T20:20:13.170592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pre-processing for the maps is done. Now, we can go and actually create the map chart.","metadata":{}},{"cell_type":"code","source":"# Create trace and add markers to map\ntrace = go.Scattermapbox(lat=df_map.latitude, \n                         lon=df_map.longitude,\n                         mode='markers',\n                         marker=dict(size=10,\n                                     color=df_map['color'],\n                                     opacity=0.8),\n                         hovertemplate = \"\",\n                         text=df_map['text'])\n\n# Define the layout for the map\nlayout = go.Layout(\n    margin={\"l\": 0, \"r\": 0, \"t\": 0, \"b\": 0},\n    mapbox=dict(\n        style='carto-positron', # Alternative: 'open-street-map',\n        center=dict(lat=df_map.latitude.median(),\n                    lon=df_map.longitude.median()),\n        zoom=10,\n        ),\n    height=700, width=900)\n\n# Create figure\nfig = go.Figure(data=[trace], layout=layout)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:24:00.854185Z","iopub.execute_input":"2024-03-14T20:24:00.854561Z","iopub.status.idle":"2024-03-14T20:24:00.875041Z","shell.execute_reply.started":"2024-03-14T20:24:00.854533Z","shell.execute_reply":"2024-03-14T20:24:00.874068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, you can hovering the cursor over the markers will reveal some meta information about the weather station.","metadata":{}},{"cell_type":"markdown","source":"# Solution","metadata":{}},{"cell_type":"markdown","source":"Interpreting the 2nd place notebook solution","metadata":{}},{"cell_type":"markdown","source":"# Loading Libraries and Datasets","metadata":{}},{"cell_type":"code","source":"!pip install -q iterative-stratification","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:13.194066Z","iopub.execute_input":"2024-03-14T20:20:13.194470Z","iopub.status.idle":"2024-03-14T20:20:25.622517Z","shell.execute_reply.started":"2024-03-14T20:20:13.194434Z","shell.execute_reply":"2024-03-14T20:20:25.621136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport os\nimport optuna\nimport holidays\nimport copy\n\nfrom category_encoders import OneHotEncoder, MEstimateEncoder, CatBoostEncoder, BinaryEncoder\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold, RepeatedMultilabelStratifiedKFold\nfrom sklearn.cluster import KMeans\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.model_selection import StratifiedKFold, RepeatedStratifiedKFold, TimeSeriesSplit\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier, HistGradientBoostingClassifier\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.impute import KNNImputer, SimpleImputer\nfrom sklearn.metrics import accuracy_score, roc_auc_score, roc_curve, make_scorer\nfrom sklearn.metrics.pairwise import euclidean_distances\nfrom sklearn.naive_bayes import GaussianNB, BernoulliNB\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.base import BaseEstimator, TransformerMixin, ClassifierMixin, clone\nfrom sklearn.multioutput import ClassifierChain\nfrom sklearn.preprocessing import FunctionTransformer, StandardScaler, PowerTransformer, LabelEncoder\nfrom sklearn.linear_model import LogisticRegression, RidgeClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.inspection import permutation_importance\nfrom scipy.cluster.hierarchy import dendrogram, linkage\nfrom scipy.spatial.distance import squareform\nfrom xgboost import XGBClassifier, XGBRFClassifier\nfrom lightgbm import LGBMClassifier\nfrom catboost import CatBoostClassifier\nfrom sklearn.feature_selection import SequentialFeatureSelector\n\nsns.set_theme(style = 'white', palette = 'viridis')\npal = sns.color_palette('viridis')\n\npd.set_option('display.max_rows', 100)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:25.624473Z","iopub.execute_input":"2024-03-14T20:20:25.625411Z","iopub.status.idle":"2024-03-14T20:20:25.641659Z","shell.execute_reply.started":"2024-03-14T20:20:25.625373Z","shell.execute_reply":"2024-03-14T20:20:25.640686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor = [\n    pl.col('dT').cast(pl.Float32),\n    \n    pl.col('ts_oxygen_saturation').cast(pl.Float32),\n    pl.col('ts_resting_heart_rate').cast(pl.Float32),\n    \n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('time_year').cast(pl.UInt16),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('time_month').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('time_week').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('time_weekday').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('time_day_of_year').cast(pl.UInt16),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('time_day').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('time_hour').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('time_minute').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('time_second').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('time_micro').cast(pl.UInt32),\n    \n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('loc_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('loc_month').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('loc_week').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('loc_weekday').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('loc_day_of_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('loc_day').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('loc_hour').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('loc_minute').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('loc_second').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('loc_micro').cast(pl.UInt32),\n    \n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('start_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('start_month').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('start_week').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('start_weekday').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('start_day_of_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('start_day').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('start_hour').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('start_minute').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('start_second').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('start_micro').cast(pl.UInt32)\n]\n\npreprocessor_2 = [\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z'),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.year().alias('year').cast(pl.UInt16),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.month().alias('month').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.day().alias('day').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.hour().alias('hour').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.minute().alias('minute').cast(pl.UInt8),\n    \n    pl.col('value').cast(pl.Float32)\n]\n\nduration_calculator = [\n    (pl.duration(days = pl.col('time_day_of_year'), hours = pl.col('time_hour'), minutes = pl.col('time_minute'), seconds = pl.col('time_second'), microseconds = pl.col('time_micro')) -\n    pl.duration(days = pl.col('start_day_of_year'), hours = pl.col('start_hour'), minutes = pl.col('start_minute'), seconds = pl.col('start_second'), microseconds = pl.col('start_micro'))).dt.total_microseconds().alias('time_start_difference'),\n    (pl.duration(days = pl.col('time_day_of_year'), hours = pl.col('time_hour'), minutes = pl.col('time_minute'), seconds = pl.col('time_second'), microseconds = pl.col('time_micro')) -\n    pl.duration(days = pl.col('loc_day_of_year'), hours = pl.col('loc_hour'), minutes = pl.col('loc_minute'), seconds = pl.col('loc_second'), microseconds = pl.col('loc_micro'))).dt.total_microseconds().alias('time_loc_difference'),\n    (pl.duration(days = pl.col('loc_day_of_year'), hours = pl.col('loc_hour'), minutes = pl.col('loc_minute'), seconds = pl.col('loc_second'), microseconds = pl.col('loc_micro')) -\n     pl.duration(days = pl.col('start_day_of_year'), hours = pl.col('start_hour'), minutes = pl.col('start_minute'), seconds = pl.col('start_second'), microseconds = pl.col('start_micro'))).dt.total_microseconds().alias('start_loc_difference')\n]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:25.642974Z","iopub.execute_input":"2024-03-14T20:20:25.643893Z","iopub.status.idle":"2024-03-14T20:20:25.682493Z","shell.execute_reply.started":"2024-03-14T20:20:25.643854Z","shell.execute_reply":"2024-03-14T20:20:25.681600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv').\\\n        with_columns(preprocessor).with_columns(duration_calculator)\ntest = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv').\\\n       with_columns(preprocessor).with_columns(duration_calculator)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:25.683645Z","iopub.execute_input":"2024-03-14T20:20:25.684033Z","iopub.status.idle":"2024-03-14T20:20:25.700179Z","shell.execute_reply.started":"2024-03-14T20:20:25.684002Z","shell.execute_reply":"2024-03-14T20:20:25.699248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Separating the data for training and testing of the model.","metadata":{}},{"cell_type":"code","source":"air_temp = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_air_temp'),\n               pl.col('value').std().alias('std_air_temp'),\n           )\nrainfall = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_rainfall'),\n               pl.col('value').std().alias('std_rainfall'),\n           )\nhumidity = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_humidity'),\n               pl.col('value').std().alias('std_humidity'),\n           )\nstations = pl.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv')\nwind_direction = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv').\\\n                 melt('').\\\n                 with_columns(preprocessor_2).\\\n                 group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n                 agg(\n                     pl.col('value').mean().alias('mean_wind_direction'),\n                     pl.col('value').std().alias('std_wind_direction')\n                 )\nwind_speed = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv').\\\n             melt('').\\\n             with_columns(preprocessor_2).\\\n             group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n             agg(\n                 pl.col('value').mean().alias('mean_wind_speed'),\n                 pl.col('value').std().alias('std_wind_speed'),\n             )","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:25.701437Z","iopub.execute_input":"2024-03-14T20:20:25.701748Z","iopub.status.idle":"2024-03-14T20:20:25.722128Z","shell.execute_reply.started":"2024-03-14T20:20:25.701725Z","shell.execute_reply":"2024-03-14T20:20:25.721270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Information","metadata":{}},{"cell_type":"code","source":"def desc_dataset_pl(df):\n    return pl.concat([\n        pl.DataFrame({'key' : df.columns, 'dataset_length' : len(df)}),\n        df.null_count().transpose().rename({'column_0' : 'null'}),\n        df.approx_n_unique().transpose().rename({'column_0' : 'nunique'}),\n        df.mean().transpose().rename({'column_0' : 'mean'}),\n        df.std().transpose().rename({'column_0' : 'std'}),\n        df.median().transpose().rename({'column_0' : 'median'}),\n        df.min().transpose().rename({'column_0' : 'min'}),\n        df.max().transpose().rename({'column_0' : 'max'})\n    ], how = 'horizontal').with_columns(\n        (pl.col('dataset_length') - pl.col('null')).alias('count'),\n        (pl.col('null') / pl.col('dataset_length') * 100).alias('%null'),\n        (pl.col('nunique') / pl.col('dataset_length') * 100).alias('%unique')\n    ).drop('dataset_length').to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:25.723268Z","iopub.execute_input":"2024-03-14T20:20:25.725526Z","iopub.status.idle":"2024-03-14T20:20:25.733657Z","shell.execute_reply.started":"2024-03-14T20:20:25.725502Z","shell.execute_reply":"2024-03-14T20:20:25.732714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function creates a new data frame that has statitstic information of each column. ","metadata":{}},{"cell_type":"code","source":"#desc_dataset_pl(train)\n#desc_dataset_pl(test)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:25.734730Z","iopub.execute_input":"2024-03-14T20:20:25.735021Z","iopub.status.idle":"2024-03-14T20:20:25.741642Z","shell.execute_reply.started":"2024-03-14T20:20:25.734998Z","shell.execute_reply":"2024-03-14T20:20:25.740786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The creator of the notebook used this function to analize the data both in the train dataframe as well as the teste dataframe.","metadata":{}},{"cell_type":"markdown","source":"# TS Features Aggregation","metadata":{}},{"cell_type":"code","source":"df_all = pl.concat([train, test], how = 'vertical')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:20:25.742758Z","iopub.execute_input":"2024-03-14T20:20:25.743064Z","iopub.status.idle":"2024-03-14T20:20:25.751315Z","shell.execute_reply.started":"2024-03-14T20:20:25.743041Z","shell.execute_reply":"2024-03-14T20:20:25.750500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"survey_counter_day = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_day').cast(pl.UInt16)\n)\nsurvey_counter_hour = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_hour').cast(pl.UInt16)\n)\nsurvey_counter_minute = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_minute').cast(pl.UInt8)\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rest_heart_rate_agg = df_all.group_by(\n    ['time_year', 'time_week', 'id_participant']\n).agg(\n    pl.col('ts_resting_heart_rate').mean().alias('mean_resting_heart_rate'),\n    pl.col('ts_resting_heart_rate').std().alias('std_resting_heart_rate')\n)\noxygen_saturation_agg = df_all.group_by(\n    ['time_year', 'time_week', 'id_participant']\n).agg(\n    pl.col('ts_oxygen_saturation').mean().alias('mean_oxygen_saturation'),\n    pl.col('ts_oxygen_saturation').std().alias('std_oxygen_saturation')\n)\nstand_time_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_stand_time').mean().alias('mean_stand_time'),\n    pl.col('ts_stand_time').std().alias('std_stand_time')\n)\nstep_count_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_step_count').mean().alias('mean_step_count'),\n    pl.col('ts_step_count').std().alias('std_step_count')\n)\nwalking_distance_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_walking_distance').mean().alias('mean_walking_distance'),\n    pl.col('ts_walking_distance').std().alias('std_walking_distance')\n)\naudio_exposure_environment_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_audio_exposure_environment').mean().alias('mean_audio_exposure_environment'),\n    pl.col('ts_audio_exposure_environment').std().alias('std_audio_exposure_environment')\n)\nheart_rate_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_heart_rate').mean().alias('mean_heart_rate'),\n    pl.col('ts_heart_rate').std().alias('std_heart_rate')\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Aggregations are made for data summarization. Each aggregation operation calculates summary statistics (mean and standard deviation) for specific health-related metrics over different time intervals.","metadata":{}},{"cell_type":"code","source":"train = train.join(\n    rest_heart_rate_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    oxygen_saturation_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    stand_time_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    step_count_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    walking_distance_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    audio_exposure_environment_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    heart_rate_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    survey_counter_day, on = ['id_participant', 'time_year', 'time_month', 'time_day'], how = 'left'\n).join(\n    survey_counter_hour, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour'], how = 'left'\n).join(\n    survey_counter_minute, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'], how = 'left'\n)\ntest = test.join(\n    rest_heart_rate_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    oxygen_saturation_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    stand_time_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    step_count_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    walking_distance_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    audio_exposure_environment_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    heart_rate_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    survey_counter_day, on = ['id_participant', 'time_year', 'time_month', 'time_day'], how = 'left'\n).join(\n    survey_counter_hour, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour'], how = 'left'\n).join(\n    survey_counter_minute, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'], how = 'left'\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then this aggregations are merged in the datasets for testing and training.","metadata":{}},{"cell_type":"markdown","source":"# Station and Weather","metadata":{}},{"cell_type":"code","source":"def find_closest_station(x):\n    if x[0] == None: return None\n    latitude = float(x[0][:18])\n    longitude = float(x[0][19:])\n    return stations.row(stations.select(\n        distance = (pl.col('latitude') - latitude) ** 2 + (pl.col('longitude') - longitude) ** 2\n    )['distance'].arg_min())[1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_station(df):\n    return df.select(pl.col('ws_latitude').cast(pl.Utf8).map_elements(\n        lambda x: x + '0' * (18 - len(x))\n    ) + ' ' + pl.col('ws_longitude').cast(pl.Utf8).map_elements(\n        lambda x: x + '0' * (18 - len(x))\n    )).map_rows(find_closest_station)['map']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.join(\n    air_temp,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    rainfall,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    humidity,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_direction,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_speed,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute'])\n\ntest = test.join(\n    air_temp,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    rainfall,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    humidity,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_direction,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_speed,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code is integrating weather-related data into both the training and test datasets based on common time-related columns, which likely serves the purpose of enhancing the predictive capabilities of a machine learning model","metadata":{}},{"cell_type":"markdown","source":"# Correlation","metadata":{}},{"cell_type":"code","source":"def heatmap(dataset, label = None):\n    corr = dataset.corr(method = 'pearson')\n    plt.figure(figsize = (45, 45), dpi = 300)\n    mask = np.zeros_like(corr)\n    mask[np.triu_indices_from(mask)] = True\n    sns.heatmap(corr, mask = mask, cmap = 'viridis', annot = True, annot_kws = {'size' : 11})\n    plt.title(f'{label} Dataset Correlation Matrix\\n', fontsize = 25, weight = 'bold')\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A function was created to plot a heatmap in order to visualize the correlation between the different types of data","metadata":{}},{"cell_type":"markdown","source":"# Preparation","metadata":{}},{"cell_type":"code","source":"X = train.filter(pl.col('q_location').is_not_null()).drop('id_unique', 'time', 'ws_timestamp_location', 'ws_timestamp_start', 'ts_oxygen_saturation', 'ts_resting_heart_rate', 'ts_stand_time', 'ts_step_count', 'ts_walking_distance', 'ts_heart_rate', 'ts_audio_exposure_environment').collect()\nX = X.with_columns(stations = map_station(X)).to_pandas()\ny = X[['q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference']]\nX = X.drop(['q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference'], axis = 1)\n\nsubmission = test.filter(pl.col('q_location').is_not_null()).drop('q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference', 'time', 'ws_timestamp_location', 'ws_timestamp_start', 'ts_oxygen_saturation', 'ts_resting_heart_rate', 'ts_stand_time', 'ts_step_count', 'ts_walking_distance', 'ts_heart_rate', 'ts_audio_exposure_environment').collect()\nsubmission = submission.with_columns(stations = map_station(submission)).to_pandas()\ntest_pd = submission.drop('id_unique', axis = 1)\ntest_pd.loc[3178, 'time_loc_difference'] += 1e6 * 3600 * 24 * 365\ntest_pd.loc[3178, 'start_loc_difference'] -= 1e6 * 3600 * 24 * 365\n\ncategorical_features = list(X.select_dtypes('object'))\n\nle = {}\n\nfor col in y.columns:\n    le[col] = LabelEncoder().fit(y[col])\n    y[col] = le[col].transform(y[col])\n    \nacc_scorer = make_scorer(accuracy_score)\nseed = 42\ntf.keras.utils.set_random_seed(seed)\n\nkf = MultilabelStratifiedKFold(n_splits = 5, random_state = seed, shuffle = True)\n#kf = TimeSeriesSplit()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this part, the filtering and dropping columns for training Data takes place. Also the user applies the function map_station() to the filtered training data to add a new column stations containing information mapped from the dataset.\n\nFeatures (X) are separated from the target variables (q_earphones, q_noise_kind, q_noise_nearby, q_thermal_preference). And finally, this part of the code setsup the data for a stratified k-fold cross-validation (kf) with 5 splits.","metadata":{}},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"class NumericalImputer(BaseEstimator, TransformerMixin):\n    \n    def fit(self, x, y = None):\n        self.imputer = SimpleImputer().fit(x.select_dtypes(['int', 'float']), y)\n        return self\n    \n    def transform(self, x):\n        x_copy = x.copy()\n        x_copy.loc[:, list(x_copy.select_dtypes(['int', 'float']))] = self.imputer.transform(x_copy.loc[:, list(x_copy.select_dtypes(['int', 'float']))])\n        return x_copy","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The purpose of this transformer is to impute missing numerical values in the dataset.","metadata":{}},{"cell_type":"markdown","source":"# Adversarial validation","metadata":{}},{"cell_type":"code","source":"#thanks to @carlmcbrideellis\n#https://www.kaggle.com/code/carlmcbrideellis/what-is-adversarial-validation\n\ndef adversarial_validation(dataset_1 = train, dataset_2 = test, label = 'Train-Test'):\n\n    adv_train = dataset_1.copy()\n    adv_test = dataset_2.copy()\n\n    adv_train['is_test'] = 0\n    adv_test['is_test'] = 1\n\n    adv = pd.concat([adv_train, adv_test], ignore_index = True)\n\n    adv_shuffled = adv.sample(frac = 1)\n\n    adv_X = adv_shuffled.drop('is_test', axis = 1)\n    adv_y = adv_shuffled.is_test\n\n    skf = StratifiedKFold(n_splits = 5, random_state = 42, shuffle = True)\n\n    val_scores = []\n    predictions = np.zeros(len(adv))\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(adv_X, adv_y)):\n    \n        adv_lr = make_pipeline(\n            MEstimateEncoder(cols = list(categorical_features)), \n            #FeatureDropper(cols = ['ws_survey_count', 'id_participant']),\n            SimpleImputer(),\n            XGBClassifier(random_state = seed, tree_method = 'hist')\n        )\n        adv_lr.fit(adv_X.iloc[train_idx], adv_y.iloc[train_idx])\n        \n        val_preds = adv_lr.predict_proba(adv_X.iloc[val_idx])[:, 1]\n        predictions[val_idx] = val_preds\n        val_score = roc_auc_score(adv_y.iloc[val_idx], val_preds)\n        val_scores.append(val_score)\n    \n    fpr, tpr, _ = roc_curve(adv['is_test'], predictions)\n    \n    plt.figure(figsize = (10, 10), dpi = 300)\n    sns.lineplot(x=[0, 1], y=[0, 1], linestyle=\"--\", label=\"Indistinguishable Datasets\")\n    sns.lineplot(x=fpr, y=tpr, label=\"Adversarial Validation Classifier\")\n    plt.title(f'{label} Validation = {np.mean(val_scores):.5f}', weight = 'bold', size = 17)\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function visualizes how well the distributions of two datasets match, which is useful for determining if the test dataset is significantly different from the training dataset. It employs a machine learning model to classify whether each data point belongs to the training or test dataset, based on the extracted features.","metadata":{}},{"cell_type":"markdown","source":"# Cross validation","metadata":{}},{"cell_type":"code","source":"def cross_val_score(estimator, cv = kf, label = '', show_importance = False):\n    \n    val_scores = []\n    \n    feature_importances_table = pd.DataFrame({'value' : 0}, index = list(X.columns))\n    \n    for fold, (train_idx, val_idx) in enumerate(cv.split(X, y)):\n        \n        X_train = X.iloc[train_idx]\n        y_train = y.iloc[train_idx]\n        \n        X_val = X.iloc[val_idx]\n        y_val = y.iloc[val_idx]\n        \n        val_preds = np.zeros((len(X_val), 4))\n        val_score = 0\n        \n        for i in range(4):\n            model = clone(estimator).fit(X_train, y_train.iloc[:, i])\n            val_preds[:, i] = model.predict(X_val)\n            val_score += accuracy_score(y_val.iloc[:, i], val_preds[:, i]) / 4\n            if show_importance:\n                feature_importances_table['value'] += permutation_importance(model, X_val, y_val.iloc[:, i], random_state = seed, scoring = 'accuracy', n_repeats = 5).importances_mean / 4 / cv.get_n_splits()\n            \n        val_scores.append(val_score)\n    \n    print(f'Val Score: {np.mean(val_scores):.5f} ± {np.std(val_scores):.5f} | {label}')\n    \n    if show_importance:\n        plt.figure(figsize = (20, 50))\n        plt.title('Feature Importances', weight = 'bold', size = 25)\n        sns.barplot(feature_importances_table.sort_values('value', ascending = False).T, orient = 'h', palette = 'viridis')\n        plt.show()\n    \n    return val_scores, feature_importances_table","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:40:07.255026Z","iopub.execute_input":"2024-03-14T20:40:07.255426Z","iopub.status.idle":"2024-03-14T20:40:08.213127Z","shell.execute_reply.started":"2024-03-14T20:40:07.255398Z","shell.execute_reply":"2024-03-14T20:40:08.211732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cross_val_score_proba(estimator, col, cv = kf, label = ''):\n    \n    val_scores = []\n    val_predictions = np.zeros((len(X), y.loc[:, col].nunique()))\n    test_predictions = np.zeros((len(test_pd), y.loc[:, col].nunique()))\n    \n    for fold, (train_idx, val_idx) in enumerate(cv.split(X, y)):\n        \n        X_train = X.iloc[train_idx].reset_index(drop = True)\n        y_train = y.iloc[train_idx].reset_index(drop = True)\n        \n        X_val = X.iloc[val_idx].reset_index(drop = True)\n        y_val = y.iloc[val_idx].reset_index(drop = True)\n        \n        model = clone(estimator).fit(X_train, y_train[col])\n        val_preds = model.predict_proba(X_val)\n        val_score = accuracy_score(y_val[col], val_preds.argmax(axis = 1))\n            \n        val_scores.append(val_score)\n        val_predictions[val_idx] = val_preds\n        test_predictions += model.predict_proba(test_pd) / cv.get_n_splits()\n        \n        #print(f'Fold Score: {val_score:.5f}')\n    \n    print(f'Val Score: {np.mean(val_scores):.5f} ± {np.std(val_scores):.5f} | {label}')\n    \n    return val_scores, val_predictions, test_predictions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_list = {col : [] for col in list(y)}\npredict_list = {col : [] for col in list(y)}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Logistic regresion","metadata":{}},{"cell_type":"code","source":"log_models = {}\n\nfor col in list(y):\n    log_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        StandardScaler(),\n        PowerTransformer(),\n        LogisticRegression(random_state = seed, max_iter = 10000000)\n    )\n    _, oof_preds, _ = cross_val_score_proba(log_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"this code trains logistic regression models for each target variable, performing preprocessing steps such as feature encoding, imputation, scaling, and power transformation. ","metadata":{}},{"cell_type":"markdown","source":"# SVM model","metadata":{}},{"cell_type":"code","source":"svm_models = {}\n\nfor col in list(y):\n    svm_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        StandardScaler(),\n        PowerTransformer(),\n        SVC(random_state = seed, probability = True, max_iter = 1000000)\n    )\n    _, oof_preds, _ = cross_val_score_proba(svm_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Overall, this code trains SVM models for each target variable, similar to the logistic regression models. It performs preprocessing steps such as feature encoding, imputation, scaling, and power transformation before training the SVM models. The performance of each model is evaluated using cross-validation, and out-of-fold predictions are stored for evaluation purposes.","metadata":{}},{"cell_type":"markdown","source":"# LightGBM","metadata":{}},{"cell_type":"code","source":"lgb_models = {}\n\nfor col in list(y):\n    lgb_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        LGBMClassifier(random_state = seed)\n    )\n    _, oof_preds, _ = cross_val_score_proba(lgb_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CatBoost","metadata":{}},{"cell_type":"code","source":"cb_models = {}\n\nfor col in list(y):\n    cb_models[col] = make_pipeline(\n        NumericalImputer(),\n        CatBoostClassifier(random_state = seed, verbose = 0, cat_features = categorical_features)\n    )\n    _, oof_preds, _ = cross_val_score_proba(cb_models[col], col)\n    oof_list[col].append(oof_preds)\n\nprint('')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cb_bayes_models = {}\n\nfor col in list(y):\n    cb_bayes_models[col] = make_pipeline(\n        NumericalImputer(),\n        CatBoostClassifier(random_state = seed, verbose = 0, bootstrap_type = 'Bayesian', cat_features = categorical_features)\n    )\n    _, oof_preds, _ = cross_val_score_proba(cb_bayes_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cb_bernoulli_models = {}\n\nfor col in list(y):\n    cb_bernoulli_models[col] = make_pipeline(\n        NumericalImputer(),\n        CatBoostClassifier(random_state = seed, verbose = 0, bootstrap_type = 'Bernoulli', cat_features = categorical_features)\n    )\n    _, oof_preds, _ = cross_val_score_proba(cb_bernoulli_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost model","metadata":{}},{"cell_type":"code","source":"xgb_models = {}\n\nfor col in list(y):\n    xgb_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        XGBClassifier(random_state = seed, tree_method = 'hist')\n    )\n    _, oof_preds, _ = cross_val_score_proba(xgb_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensamble NN","metadata":{}},{"cell_type":"code","source":"def cross_val_score_nn(estimator, col, cv = kf, label = ''):\n    \n    X_oof = oof_list[col]\n        \n    model = clone(estimator).fit(X_oof, y[col])\n    val_preds = model.predict_proba(X_oof)\n    val_score = accuracy_score(y[col], val_preds.argmax(axis = 1))\n    \n    print(f'Val Score: {val_score:.5f} | {label}')\n    \n    return val_score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function evaluates the performance of a neural network model using cross-validation and prints the accuracy score for each fold.","metadata":{}},{"cell_type":"code","source":"class TensorFlower(BaseEstimator, ClassifierMixin):\n    \n    def fit(self, x, y):\n        \n        tf.keras.utils.set_random_seed(seed)\n        tf.config.experimental.enable_op_determinism()\n        \n        inputs = []\n        norm_inputs = []\n        \n        multiply = []\n        \n        for i in range(len(x)):\n            inputs.append(tf.keras.Input((len(np.unique(y)), )))\n            norm_inputs.append(tf.keras.layers.BatchNormalization()(inputs[i]))\n            \n        concat = tf.keras.layers.Concatenate()(norm_inputs)\n        \n        for i in range(len(x)):\n            dense = tf.keras.layers.Dense(len(np.unique(y)))(concat)\n            norm = tf.keras.layers.BatchNormalization()(dense)\n            act = tf.keras.activations.softmax(norm)\n            multiply.append(tf.keras.layers.Multiply()([inputs[i], act]))\n            \n        outputs = tf.keras.layers.Add()(multiply)\n        \n        self.model = tf.keras.Model(inputs, outputs)\n        self.model.compile(loss = 'sparse_categorical_crossentropy', optimizer = tf.keras.optimizers.AdamW(1e-4))\n        \n        self.model.fit(x, y, epochs = 50, verbose = 0)\n        \n        return self\n    \n    def predict_proba(self, x):\n        \n        return self.model.predict(x, verbose = 0)\n    \n    def predict(self, x):\n        \n        return np.argmax(self.predict_proba(x), axis = 1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in list(y):\n    cross_val_score_nn(TensorFlower(), col)\n    \nprint('\\n')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This class implements a custom neural network classifier using TensorFlow. It defines a neural network architecture with batch normalization and softmax activation for each feature, aiming to capture complex relationships between features and target classes.","metadata":{}},{"cell_type":"markdown","source":"# Prediction and submission","metadata":{}},{"cell_type":"code","source":"for col in list(y):\n    predict_list[col].append(log_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(svm_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(lgb_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(cb_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(cb_bayes_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(cb_bernoulli_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(xgb_models[col].fit(X, y[col]).predict_proba(test_pd))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in list(y):\n    ensembler = TensorFlower().fit(oof_list[col], y[col])\n    submission[col] = ensembler.predict(predict_list[col])\n    submission[col] = le[col].inverse_transform(submission[col].astype(np.int8))\n\nsubmission[['id_unique'] + list(y)].to_csv('cool_city.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this part, the code is generating predictions for each model and submiting the challenge.","metadata":{}}]}