{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61145,"databundleVersionId":7047669,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Applications","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"This notebook has to be created through the competition","metadata":{}},{"cell_type":"markdown","source":"## Table of Contents\n* [Read data](#Read-data)\n* [Data pre-processing](#Data-pre-processing)\n* [Map of all weather stations that measure air temperature](#Map-of-all-weather-stations-that-measure-air-temperature)\n* [What's next?](#What's-next)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport math","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:26:06.059091Z","iopub.execute_input":"2024-03-14T20:26:06.059534Z","iopub.status.idle":"2024-03-14T20:26:06.066636Z","shell.execute_reply.started":"2024-03-14T20:26:06.059495Z","shell.execute_reply":"2024-03-14T20:26:06.065662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], \n                       keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                       date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0],\n                          keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                          date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:26:06.068584Z","iopub.execute_input":"2024-03-14T20:26:06.069172Z","iopub.status.idle":"2024-03-14T20:27:59.323370Z","shell.execute_reply.started":"2024-03-14T20:26:06.069138Z","shell.execute_reply":"2024-03-14T20:27:59.322465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:27:59.324668Z","iopub.execute_input":"2024-03-14T20:27:59.325055Z","iopub.status.idle":"2024-03-14T20:27:59.335514Z","shell.execute_reply.started":"2024-03-14T20:27:59.325024Z","shell.execute_reply":"2024-03-14T20:27:59.334416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data pre-processing\n\n### Check dates\n\nWhen checking the last date of different kinds of dataframes it becomes clear that the watch survey responses only last until Jan 21, 2023, while the weather data lasts until July 3, 2023.","metadata":{}},{"cell_type":"code","source":"print(df_train.index.max())\nprint(df_solution.index.max())\nprint(df_rain.index.max())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:27:59.336728Z","iopub.execute_input":"2024-03-14T20:27:59.337146Z","iopub.status.idle":"2024-03-14T20:27:59.360995Z","shell.execute_reply.started":"2024-03-14T20:27:59.337115Z","shell.execute_reply":"2024-03-14T20:27:59.359809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't really need the weather data after Jan 21, 2023, so we might just as well get rid of it.","metadata":{}},{"cell_type":"code","source":"df_rain = df_rain[df_rain.index<'2023-01-22']\ndf_temperature = df_temperature[df_temperature.index<'2023-01-22']\ndf_humidity = df_humidity[df_humidity.index<'2023-01-22']\ndf_wind_speed = df_wind_speed[df_wind_speed.index<'2023-01-22']\ndf_wind_direction = df_wind_direction[df_wind_direction.index<'2023-01-22']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:27:59.363486Z","iopub.execute_input":"2024-03-14T20:27:59.363800Z","iopub.status.idle":"2024-03-14T20:27:59.400831Z","shell.execute_reply.started":"2024-03-14T20:27:59.363765Z","shell.execute_reply":"2024-03-14T20:27:59.400001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check weather data","metadata":{}},{"cell_type":"code","source":"# Show new shortened air temperature dataframe\ndf_temperature.describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:27:59.401986Z","iopub.execute_input":"2024-03-14T20:27:59.402259Z","iopub.status.idle":"2024-03-14T20:27:59.532495Z","shell.execute_reply.started":"2024-03-14T20:27:59.402235Z","shell.execute_reply":"2024-03-14T20:27:59.531459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like there is no temperature data available in before Jan 21, 2023 for weather station S117. So, we might just as well remove that column","metadata":{}},{"cell_type":"code","source":"df_temperature = df_temperature.drop(columns=['S117'])\n# Get dimensions of the air temperature data\ndf_temperature.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:27:59.533604Z","iopub.execute_input":"2024-03-14T20:27:59.533900Z","iopub.status.idle":"2024-03-14T20:27:59.543025Z","shell.execute_reply.started":"2024-03-14T20:27:59.533874Z","shell.execute_reply":"2024-03-14T20:27:59.542086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The weather dataframes are rather large. This can make the plotting of the data slow and the output images large. To make it a bit lighter, we are going to resample the data hourly.","metadata":{"execution":{"iopub.status.busy":"2024-03-12T21:09:04.412698Z","iopub.execute_input":"2024-03-12T21:09:04.413286Z","iopub.status.idle":"2024-03-12T21:09:04.419237Z","shell.execute_reply.started":"2024-03-12T21:09:04.413253Z","shell.execute_reply":"2024-03-12T21:09:04.41811Z"}}},{"cell_type":"code","source":"# Resample weather data\ndf_temperature_average = df_temperature.resample('60min').mean()\ndf_rain_sum = df_rain.resample('60min').sum() # note: for rain the data is aggregated with a sum instead of the mean\ndf_humidity_average = df_humidity.resample('60min').mean()\ndf_wind_speed_average = df_wind_speed.resample('60min').mean()\ndf_wind_direction_average = df_wind_direction.resample('60min').mean()\n\n# Check size of resulting dataframe\ndf_temperature_average.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:27:59.544514Z","iopub.execute_input":"2024-03-14T20:27:59.544923Z","iopub.status.idle":"2024-03-14T20:27:59.628616Z","shell.execute_reply.started":"2024-03-14T20:27:59.544893Z","shell.execute_reply":"2024-03-14T20:27:59.627792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Map watch survey questions to question ID\n","metadata":{}},{"cell_type":"code","source":"ws_questions = {\"q_noise_nearby\": \"Noise distractions nearby? (without earphones)\",\n                \"q_noise_kind\": \"What kind of noise?\",\n                \"q_earphones\": \"Wearing earphones?\",\n                \"q_thermal_preference\": \"Thermally, what do you prefer now?\",\n                \"q_location\": \"Where are you?\",\n                \"q_location_office\": \"What kind of office?\",\n                \"q_location_transport\": \"What kind of transport?\",\n                \"q_alone_group\": \"Alone or in a group?\",\n                \"q_activity_category_alone\": \"Category of activity? (alone)\",\n                \"q_activity_category_group\": \"Category of activity? (group)\"}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:27:59.629770Z","iopub.execute_input":"2024-03-14T20:27:59.630045Z","iopub.status.idle":"2024-03-14T20:27:59.635060Z","shell.execute_reply.started":"2024-03-14T20:27:59.630021Z","shell.execute_reply":"2024-03-14T20:27:59.634005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot weather data\n\n### Plot weather weather data time-series¶\n\nBelow we are going to plot the air temperature data from all weather stations. I used the Matplotlib method plt.plot() instead of sns.lineplot(). The reason is that plt.plot() shows the gaps in the data, while sns.lineplot() draws straight lines accross the gaps.\n","metadata":{}},{"cell_type":"code","source":"cols_num = 1\nrows_num = len(df_temperature_average.columns)\ncol_current = 0\nrow_current = 0 \nfig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\nplt.subplots_adjust(hspace = 0.5)\n\nfor i, col in enumerate(df_temperature_average.columns):\n    row_current = i\n    axs[row_current].plot(df_temperature_average.index, df_temperature_average[col])\n    station_name = df_stations[df_stations['id']==col]['name'].values[0]\n    axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n    axs[row_current].set_ylabel('Air Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:27:59.636094Z","iopub.execute_input":"2024-03-14T20:27:59.636324Z","iopub.status.idle":"2024-03-14T20:28:05.193127Z","shell.execute_reply.started":"2024-03-14T20:27:59.636304Z","shell.execute_reply":"2024-03-14T20:28:05.191829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since, we are going to plot more weather data, we might just as well wrap the above code for plotting in to a function.","metadata":{}},{"cell_type":"code","source":"# Function that plots weather data fromm all weather stations\ndef plot_weather(df, ylabel):\n    cols_num = 1\n    rows_num = len(df.columns)\n    col_current = 0\n    row_current = 0 \n    fig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\n    plt.subplots_adjust(hspace = 0.5)\n\n    for i, col in enumerate(df.columns):\n        row_current = i\n        axs[row_current].plot(df.index, df[col])\n        station_name = df_stations[df_stations['id']==col]['name'].values[0]\n        axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n        axs[row_current].set_ylabel(ylabel)\n    \n    return fig, axs","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:05.194372Z","iopub.execute_input":"2024-03-14T20:28:05.194644Z","iopub.status.idle":"2024-03-14T20:28:05.201983Z","shell.execute_reply.started":"2024-03-14T20:28:05.194618Z","shell.execute_reply":"2024-03-14T20:28:05.201093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Temperature","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_temperature_average, 'Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:05.203141Z","iopub.execute_input":"2024-03-14T20:28:05.203398Z","iopub.status.idle":"2024-03-14T20:28:10.902397Z","shell.execute_reply.started":"2024-03-14T20:28:05.203375Z","shell.execute_reply":"2024-03-14T20:28:10.900955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Rainfall","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_rain_sum, 'Rainfall [mm/h]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:10.903863Z","iopub.execute_input":"2024-03-14T20:28:10.904195Z","iopub.status.idle":"2024-03-14T20:28:33.946280Z","shell.execute_reply.started":"2024-03-14T20:28:10.904166Z","shell.execute_reply":"2024-03-14T20:28:33.945141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relative humidity","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_humidity_average, 'Relative humidity [%]')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:33.953069Z","iopub.execute_input":"2024-03-14T20:28:33.953884Z","iopub.status.idle":"2024-03-14T20:28:39.099737Z","shell.execute_reply.started":"2024-03-14T20:28:33.953851Z","shell.execute_reply":"2024-03-14T20:28:39.098585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Map of all weather stations that measure air temperature","metadata":{}},{"cell_type":"code","source":"# Have a look at the weather station meta data\ndf_stations.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:39.101078Z","iopub.execute_input":"2024-03-14T20:28:39.101378Z","iopub.status.idle":"2024-03-14T20:28:39.113384Z","shell.execute_reply.started":"2024-03-14T20:28:39.101350Z","shell.execute_reply":"2024-03-14T20:28:39.112210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:39.114478Z","iopub.execute_input":"2024-03-14T20:28:39.114854Z","iopub.status.idle":"2024-03-14T20:28:39.127714Z","shell.execute_reply.started":"2024-03-14T20:28:39.114827Z","shell.execute_reply":"2024-03-14T20:28:39.126941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a look at the air temperature data\ndf_temperature.columns","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:39.128978Z","iopub.execute_input":"2024-03-14T20:28:39.129275Z","iopub.status.idle":"2024-03-14T20:28:39.139484Z","shell.execute_reply.started":"2024-03-14T20:28:39.129250Z","shell.execute_reply":"2024-03-14T20:28:39.138420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, the list of all weather stations is much longer then the list with weather stations that can measure air temperature. Hence, we need to pre-process the data a bit and only pick the weather stations that can measure temperature data.","metadata":{}},{"cell_type":"code","source":"# Initialize a new dataframe for creating the map\ndf_map = pd.DataFrame(columns=['latitude', 'longitude', 'text'])\n\n\n# Go through each column of the temperature dataframe\nfor col in df_temperature.columns:\n    # Get the weather station name based on the station id\n    station_name = df_stations.loc[df_stations.id==col].name.values[0]\n    \n    # Create a new row, with latitude, longitude, hover text, and marker color as columns\n    df_row = pd.DataFrame([{'latitude': df_stations.loc[df_stations.id==col].latitude.values[0], \n                            'longitude': df_stations.loc[df_stations.id==col].longitude.values[0], \n                            'text': f'Station ID: {col} <br>Station name: {station_name} <br>Measurement: Rain',\n                            'color': 'rgb(0, 128, 255)'}])\n    \n    # Append new row to map dataframe\n    df_map = pd.concat([df_map, df_row])\n    \ndf_map.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:39.140860Z","iopub.execute_input":"2024-03-14T20:28:39.141394Z","iopub.status.idle":"2024-03-14T20:28:39.196092Z","shell.execute_reply.started":"2024-03-14T20:28:39.141350Z","shell.execute_reply":"2024-03-14T20:28:39.195169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pre-processing for the maps is done. Now, we can go and actually create the map chart.","metadata":{}},{"cell_type":"code","source":"# Create trace and add markers to map\ntrace = go.Scattermapbox(lat=df_map.latitude, \n                         lon=df_map.longitude,\n                         mode='markers',\n                         marker=dict(size=10,\n                                     color=df_map['color'],\n                                     opacity=0.8),\n                         hovertemplate = \"\",\n                         text=df_map['text'])\n\n# Define the layout for the map\nlayout = go.Layout(\n    margin={\"l\": 0, \"r\": 0, \"t\": 0, \"b\": 0},\n    mapbox=dict(\n        style='carto-positron', # Alternative: 'open-street-map',\n        center=dict(lat=df_map.latitude.median(),\n                    lon=df_map.longitude.median()),\n        zoom=10,\n        ),\n    height=700, width=900)\n\n# Create figure\nfig = go.Figure(data=[trace], layout=layout)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:39.197246Z","iopub.execute_input":"2024-03-14T20:28:39.197561Z","iopub.status.idle":"2024-03-14T20:28:39.675295Z","shell.execute_reply.started":"2024-03-14T20:28:39.197535Z","shell.execute_reply":"2024-03-14T20:28:39.674064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, you can hovering the cursor over the markers will reveal some meta information about the weather station.","metadata":{}},{"cell_type":"markdown","source":"## What's next?","metadata":{}},{"cell_type":"markdown","source":"How do I continue this? Where to start?!\n\nGO TO THE COMPETITION SITE!\n\n[https://www.kaggle.com/competitions/cool-quiet-city-competition](https://www.kaggle.com/competitions/cool-quiet-city-competition)","metadata":{}},{"cell_type":"markdown","source":"# Tomando el código del segundo puesto","metadata":{}},{"cell_type":"code","source":"!pip install -q iterative-stratification","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:39.676916Z","iopub.execute_input":"2024-03-14T20:28:39.677278Z","iopub.status.idle":"2024-03-14T20:28:53.753973Z","shell.execute_reply.started":"2024-03-14T20:28:39.677244Z","shell.execute_reply":"2024-03-14T20:28:53.752727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport os\nimport optuna\nimport holidays\nimport copy\n\nfrom category_encoders import OneHotEncoder, MEstimateEncoder, CatBoostEncoder, BinaryEncoder\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold, RepeatedMultilabelStratifiedKFold\nfrom sklearn.cluster import KMeans\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.model_selection import StratifiedKFold, RepeatedStratifiedKFold, TimeSeriesSplit\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier, HistGradientBoostingClassifier\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.impute import KNNImputer, SimpleImputer\nfrom sklearn.metrics import accuracy_score, roc_auc_score, roc_curve, make_scorer\nfrom sklearn.metrics.pairwise import euclidean_distances\nfrom sklearn.naive_bayes import GaussianNB, BernoulliNB\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.base import BaseEstimator, TransformerMixin, ClassifierMixin, clone\nfrom sklearn.multioutput import ClassifierChain\nfrom sklearn.preprocessing import FunctionTransformer, StandardScaler, PowerTransformer, LabelEncoder\nfrom sklearn.linear_model import LogisticRegression, RidgeClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.inspection import permutation_importance\nfrom scipy.cluster.hierarchy import dendrogram, linkage\nfrom scipy.spatial.distance import squareform\nfrom xgboost import XGBClassifier, XGBRFClassifier\nfrom lightgbm import LGBMClassifier\nfrom catboost import CatBoostClassifier\nfrom sklearn.feature_selection import SequentialFeatureSelector\n\nsns.set_theme(style = 'white', palette = 'viridis')\npal = sns.color_palette('viridis')\n\npd.set_option('display.max_rows', 100)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:28:53.757024Z","iopub.execute_input":"2024-03-14T20:28:53.757340Z","iopub.status.idle":"2024-03-14T20:29:12.058410Z","shell.execute_reply.started":"2024-03-14T20:28:53.757312Z","shell.execute_reply":"2024-03-14T20:29:12.057552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor = [\n    pl.col('dT').cast(pl.Float32),\n    \n    pl.col('ts_oxygen_saturation').cast(pl.Float32),\n    pl.col('ts_resting_heart_rate').cast(pl.Float32),\n    \n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('time_year').cast(pl.UInt16),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('time_month').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('time_week').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('time_weekday').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('time_day_of_year').cast(pl.UInt16),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('time_day').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('time_hour').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('time_minute').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('time_second').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('time_micro').cast(pl.UInt32),\n    \n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('loc_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('loc_month').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('loc_week').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('loc_weekday').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('loc_day_of_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('loc_day').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('loc_hour').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('loc_minute').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('loc_second').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('loc_micro').cast(pl.UInt32),\n    \n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('start_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('start_month').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('start_week').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('start_weekday').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('start_day_of_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('start_day').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('start_hour').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('start_minute').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('start_second').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('start_micro').cast(pl.UInt32)\n]\n\npreprocessor_2 = [\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z'),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.year().alias('year').cast(pl.UInt16),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.month().alias('month').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.day().alias('day').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.hour().alias('hour').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.minute().alias('minute').cast(pl.UInt8),\n    \n    pl.col('value').cast(pl.Float32)\n]\n\nduration_calculator = [\n    (pl.duration(days = pl.col('time_day_of_year'), hours = pl.col('time_hour'), minutes = pl.col('time_minute'), seconds = pl.col('time_second'), microseconds = pl.col('time_micro')) -\n    pl.duration(days = pl.col('start_day_of_year'), hours = pl.col('start_hour'), minutes = pl.col('start_minute'), seconds = pl.col('start_second'), microseconds = pl.col('start_micro'))).dt.total_microseconds().alias('time_start_difference'),\n    (pl.duration(days = pl.col('time_day_of_year'), hours = pl.col('time_hour'), minutes = pl.col('time_minute'), seconds = pl.col('time_second'), microseconds = pl.col('time_micro')) -\n    pl.duration(days = pl.col('loc_day_of_year'), hours = pl.col('loc_hour'), minutes = pl.col('loc_minute'), seconds = pl.col('loc_second'), microseconds = pl.col('loc_micro'))).dt.total_microseconds().alias('time_loc_difference'),\n    (pl.duration(days = pl.col('loc_day_of_year'), hours = pl.col('loc_hour'), minutes = pl.col('loc_minute'), seconds = pl.col('loc_second'), microseconds = pl.col('loc_micro')) -\n     pl.duration(days = pl.col('start_day_of_year'), hours = pl.col('start_hour'), minutes = pl.col('start_minute'), seconds = pl.col('start_second'), microseconds = pl.col('start_micro'))).dt.total_microseconds().alias('start_loc_difference')\n]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.059776Z","iopub.execute_input":"2024-03-14T20:29:12.060092Z","iopub.status.idle":"2024-03-14T20:29:12.115901Z","shell.execute_reply.started":"2024-03-14T20:29:12.060065Z","shell.execute_reply":"2024-03-14T20:29:12.115006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv').\\\n        with_columns(preprocessor).with_columns(duration_calculator)\ntest = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv').\\\n       with_columns(preprocessor).with_columns(duration_calculator)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.117131Z","iopub.execute_input":"2024-03-14T20:29:12.117378Z","iopub.status.idle":"2024-03-14T20:29:12.179326Z","shell.execute_reply.started":"2024-03-14T20:29:12.117358Z","shell.execute_reply":"2024-03-14T20:29:12.178611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"air_temp = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_air_temp'),\n               pl.col('value').std().alias('std_air_temp'),\n           )\nrainfall = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_rainfall'),\n               pl.col('value').std().alias('std_rainfall'),\n           )\nhumidity = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_humidity'),\n               pl.col('value').std().alias('std_humidity'),\n           )\nstations = pl.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv')\nwind_direction = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv').\\\n                 melt('').\\\n                 with_columns(preprocessor_2).\\\n                 group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n                 agg(\n                     pl.col('value').mean().alias('mean_wind_direction'),\n                     pl.col('value').std().alias('std_wind_direction')\n                 )\nwind_speed = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv').\\\n             melt('').\\\n             with_columns(preprocessor_2).\\\n             group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n             agg(\n                 pl.col('value').mean().alias('mean_wind_speed'),\n                 pl.col('value').std().alias('std_wind_speed'),\n             )","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.180627Z","iopub.execute_input":"2024-03-14T20:29:12.180907Z","iopub.status.idle":"2024-03-14T20:29:12.221192Z","shell.execute_reply.started":"2024-03-14T20:29:12.180883Z","shell.execute_reply":"2024-03-14T20:29:12.220466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Information","metadata":{}},{"cell_type":"code","source":"def desc_dataset_pl(df):\n    return pl.concat([\n        pl.DataFrame({'key' : df.columns, 'dataset_length' : len(df)}),\n        df.null_count().transpose().rename({'column_0' : 'null'}),\n        df.approx_n_unique().transpose().rename({'column_0' : 'nunique'}),\n        df.mean().transpose().rename({'column_0' : 'mean'}),\n        df.std().transpose().rename({'column_0' : 'std'}),\n        df.median().transpose().rename({'column_0' : 'median'}),\n        df.min().transpose().rename({'column_0' : 'min'}),\n        df.max().transpose().rename({'column_0' : 'max'})\n    ], how = 'horizontal').with_columns(\n        (pl.col('dataset_length') - pl.col('null')).alias('count'),\n        (pl.col('null') / pl.col('dataset_length') * 100).alias('%null'),\n        (pl.col('nunique') / pl.col('dataset_length') * 100).alias('%unique')\n    ).drop('dataset_length').to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.222164Z","iopub.execute_input":"2024-03-14T20:29:12.222440Z","iopub.status.idle":"2024-03-14T20:29:12.232044Z","shell.execute_reply.started":"2024-03-14T20:29:12.222416Z","shell.execute_reply":"2024-03-14T20:29:12.230915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TS Features Aggregation","metadata":{}},{"cell_type":"code","source":"df_all = pl.concat([train, test], how = 'vertical')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.233214Z","iopub.execute_input":"2024-03-14T20:29:12.233478Z","iopub.status.idle":"2024-03-14T20:29:12.244573Z","shell.execute_reply.started":"2024-03-14T20:29:12.233455Z","shell.execute_reply":"2024-03-14T20:29:12.243675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"survey_counter_day = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_day').cast(pl.UInt16)\n)\nsurvey_counter_hour = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_hour').cast(pl.UInt16)\n)\nsurvey_counter_minute = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_minute').cast(pl.UInt8)\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.245486Z","iopub.execute_input":"2024-03-14T20:29:12.245708Z","iopub.status.idle":"2024-03-14T20:29:12.257093Z","shell.execute_reply.started":"2024-03-14T20:29:12.245688Z","shell.execute_reply":"2024-03-14T20:29:12.256217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rest_heart_rate_agg = df_all.group_by(\n    ['time_year', 'time_week', 'id_participant']\n).agg(\n    pl.col('ts_resting_heart_rate').mean().alias('mean_resting_heart_rate'),\n    pl.col('ts_resting_heart_rate').std().alias('std_resting_heart_rate')\n)\noxygen_saturation_agg = df_all.group_by(\n    ['time_year', 'time_week', 'id_participant']\n).agg(\n    pl.col('ts_oxygen_saturation').mean().alias('mean_oxygen_saturation'),\n    pl.col('ts_oxygen_saturation').std().alias('std_oxygen_saturation')\n)\nstand_time_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_stand_time').mean().alias('mean_stand_time'),\n    pl.col('ts_stand_time').std().alias('std_stand_time')\n)\nstep_count_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_step_count').mean().alias('mean_step_count'),\n    pl.col('ts_step_count').std().alias('std_step_count')\n)\nwalking_distance_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_walking_distance').mean().alias('mean_walking_distance'),\n    pl.col('ts_walking_distance').std().alias('std_walking_distance')\n)\naudio_exposure_environment_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_audio_exposure_environment').mean().alias('mean_audio_exposure_environment'),\n    pl.col('ts_audio_exposure_environment').std().alias('std_audio_exposure_environment')\n)\nheart_rate_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_heart_rate').mean().alias('mean_heart_rate'),\n    pl.col('ts_heart_rate').std().alias('std_heart_rate')\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.258138Z","iopub.execute_input":"2024-03-14T20:29:12.258448Z","iopub.status.idle":"2024-03-14T20:29:12.273888Z","shell.execute_reply.started":"2024-03-14T20:29:12.258425Z","shell.execute_reply":"2024-03-14T20:29:12.273003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.join(\n    rest_heart_rate_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    oxygen_saturation_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    stand_time_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    step_count_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    walking_distance_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    audio_exposure_environment_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    heart_rate_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    survey_counter_day, on = ['id_participant', 'time_year', 'time_month', 'time_day'], how = 'left'\n).join(\n    survey_counter_hour, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour'], how = 'left'\n).join(\n    survey_counter_minute, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'], how = 'left'\n)\ntest = test.join(\n    rest_heart_rate_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    oxygen_saturation_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    stand_time_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    step_count_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    walking_distance_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    audio_exposure_environment_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    heart_rate_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    survey_counter_day, on = ['id_participant', 'time_year', 'time_month', 'time_day'], how = 'left'\n).join(\n    survey_counter_hour, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour'], how = 'left'\n).join(\n    survey_counter_minute, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'], how = 'left'\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.275000Z","iopub.execute_input":"2024-03-14T20:29:12.275292Z","iopub.status.idle":"2024-03-14T20:29:12.298345Z","shell.execute_reply.started":"2024-03-14T20:29:12.275269Z","shell.execute_reply":"2024-03-14T20:29:12.297447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Station and Weather","metadata":{}},{"cell_type":"code","source":"def find_closest_station(x):\n    if x[0] == None: return None\n    latitude = float(x[0][:18])\n    longitude = float(x[0][19:])\n    return stations.row(stations.select(\n        distance = (pl.col('latitude') - latitude) ** 2 + (pl.col('longitude') - longitude) ** 2\n    )['distance'].arg_min())[1]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.299635Z","iopub.execute_input":"2024-03-14T20:29:12.299930Z","iopub.status.idle":"2024-03-14T20:29:12.310995Z","shell.execute_reply.started":"2024-03-14T20:29:12.299907Z","shell.execute_reply":"2024-03-14T20:29:12.310227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_station(df):\n    return df.select(pl.col('ws_latitude').cast(pl.Utf8).map_elements(\n        lambda x: x + '0' * (18 - len(x))\n    ) + ' ' + pl.col('ws_longitude').cast(pl.Utf8).map_elements(\n        lambda x: x + '0' * (18 - len(x))\n    )).map_rows(find_closest_station)['map']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.311898Z","iopub.execute_input":"2024-03-14T20:29:12.312208Z","iopub.status.idle":"2024-03-14T20:29:12.324324Z","shell.execute_reply.started":"2024-03-14T20:29:12.312180Z","shell.execute_reply":"2024-03-14T20:29:12.323696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.join(\n    air_temp,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    rainfall,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    humidity,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_direction,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_speed,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute'])\n\ntest = test.join(\n    air_temp,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    rainfall,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    humidity,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_direction,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_speed,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.325636Z","iopub.execute_input":"2024-03-14T20:29:12.325914Z","iopub.status.idle":"2024-03-14T20:29:12.354295Z","shell.execute_reply.started":"2024-03-14T20:29:12.325892Z","shell.execute_reply":"2024-03-14T20:29:12.353311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation","metadata":{}},{"cell_type":"code","source":"def heatmap(dataset, label = None):\n    corr = dataset.corr(method = 'pearson')\n    plt.figure(figsize = (45, 45), dpi = 300)\n    mask = np.zeros_like(corr)\n    mask[np.triu_indices_from(mask)] = True\n    sns.heatmap(corr, mask = mask, cmap = 'viridis', annot = True, annot_kws = {'size' : 11})\n    plt.title(f'{label} Dataset Correlation Matrix\\n', fontsize = 25, weight = 'bold')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.355556Z","iopub.execute_input":"2024-03-14T20:29:12.355912Z","iopub.status.idle":"2024-03-14T20:29:12.367476Z","shell.execute_reply.started":"2024-03-14T20:29:12.355879Z","shell.execute_reply":"2024-03-14T20:29:12.366655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation","metadata":{}},{"cell_type":"code","source":"X = train.filter(pl.col('q_location').is_not_null()).drop('id_unique', 'time', 'ws_timestamp_location', 'ws_timestamp_start', 'ts_oxygen_saturation', 'ts_resting_heart_rate', 'ts_stand_time', 'ts_step_count', 'ts_walking_distance', 'ts_heart_rate', 'ts_audio_exposure_environment').collect()\nX = X.with_columns(stations = map_station(X)).to_pandas()\ny = X[['q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference']]\nX = X.drop(['q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference'], axis = 1)\n\nsubmission = test.filter(pl.col('q_location').is_not_null()).drop('q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference', 'time', 'ws_timestamp_location', 'ws_timestamp_start', 'ts_oxygen_saturation', 'ts_resting_heart_rate', 'ts_stand_time', 'ts_step_count', 'ts_walking_distance', 'ts_heart_rate', 'ts_audio_exposure_environment').collect()\nsubmission = submission.with_columns(stations = map_station(submission)).to_pandas()\ntest_pd = submission.drop('id_unique', axis = 1)\ntest_pd.loc[3178, 'time_loc_difference'] += 1e6 * 3600 * 24 * 365\ntest_pd.loc[3178, 'start_loc_difference'] -= 1e6 * 3600 * 24 * 365\n\ncategorical_features = list(X.select_dtypes('object'))\n\nle = {}\n\nfor col in y.columns:\n    le[col] = LabelEncoder().fit(y[col])\n    y[col] = le[col].transform(y[col])\n    \nacc_scorer = make_scorer(accuracy_score)\nseed = 42\ntf.keras.utils.set_random_seed(seed)\n\nkf = MultilabelStratifiedKFold(n_splits = 5, random_state = seed, shuffle = True)\n#kf = TimeSeriesSplit()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:12.368471Z","iopub.execute_input":"2024-03-14T20:29:12.368727Z","iopub.status.idle":"2024-03-14T20:29:48.082132Z","shell.execute_reply.started":"2024-03-14T20:29:12.368705Z","shell.execute_reply":"2024-03-14T20:29:48.081132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Featuring Engineering","metadata":{}},{"cell_type":"code","source":"class NumericalImputer(BaseEstimator, TransformerMixin):\n    \n    def fit(self, x, y = None):\n        self.imputer = SimpleImputer().fit(x.select_dtypes(['int', 'float']), y)\n        return self\n    \n    def transform(self, x):\n        x_copy = x.copy()\n        x_copy.loc[:, list(x_copy.select_dtypes(['int', 'float']))] = self.imputer.transform(x_copy.loc[:, list(x_copy.select_dtypes(['int', 'float']))])\n        return x_copy","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:48.083427Z","iopub.execute_input":"2024-03-14T20:29:48.085017Z","iopub.status.idle":"2024-03-14T20:29:48.091434Z","shell.execute_reply.started":"2024-03-14T20:29:48.084988Z","shell.execute_reply":"2024-03-14T20:29:48.090375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adversarial Validation","metadata":{}},{"cell_type":"code","source":"#thanks to @carlmcbrideellis\n#https://www.kaggle.com/code/carlmcbrideellis/what-is-adversarial-validation\n\ndef adversarial_validation(dataset_1 = train, dataset_2 = test, label = 'Train-Test'):\n\n    adv_train = dataset_1.copy()\n    adv_test = dataset_2.copy()\n\n    adv_train['is_test'] = 0\n    adv_test['is_test'] = 1\n\n    adv = pd.concat([adv_train, adv_test], ignore_index = True)\n\n    adv_shuffled = adv.sample(frac = 1)\n\n    adv_X = adv_shuffled.drop('is_test', axis = 1)\n    adv_y = adv_shuffled.is_test\n\n    skf = StratifiedKFold(n_splits = 5, random_state = 42, shuffle = True)\n\n    val_scores = []\n    predictions = np.zeros(len(adv))\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(adv_X, adv_y)):\n    \n        adv_lr = make_pipeline(\n            MEstimateEncoder(cols = list(categorical_features)), \n            #FeatureDropper(cols = ['ws_survey_count', 'id_participant']),\n            SimpleImputer(),\n            XGBClassifier(random_state = seed, tree_method = 'hist')\n        )\n        adv_lr.fit(adv_X.iloc[train_idx], adv_y.iloc[train_idx])\n        \n        val_preds = adv_lr.predict_proba(adv_X.iloc[val_idx])[:, 1]\n        predictions[val_idx] = val_preds\n        val_score = roc_auc_score(adv_y.iloc[val_idx], val_preds)\n        val_scores.append(val_score)\n    \n    fpr, tpr, _ = roc_curve(adv['is_test'], predictions)\n    \n    plt.figure(figsize = (10, 10), dpi = 300)\n    sns.lineplot(x=[0, 1], y=[0, 1], linestyle=\"--\", label=\"Indistinguishable Datasets\")\n    sns.lineplot(x=fpr, y=tpr, label=\"Adversarial Validation Classifier\")\n    plt.title(f'{label} Validation = {np.mean(val_scores):.5f}', weight = 'bold', size = 17)\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:48.092930Z","iopub.execute_input":"2024-03-14T20:29:48.093302Z","iopub.status.idle":"2024-03-14T20:29:48.105043Z","shell.execute_reply.started":"2024-03-14T20:29:48.093277Z","shell.execute_reply":"2024-03-14T20:29:48.104181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross Validation","metadata":{}},{"cell_type":"code","source":"def cross_val_score(estimator, cv = kf, label = '', show_importance = False):\n    \n    val_scores = []\n    \n    feature_importances_table = pd.DataFrame({'value' : 0}, index = list(X.columns))\n    \n    for fold, (train_idx, val_idx) in enumerate(cv.split(X, y)):\n        \n        X_train = X.iloc[train_idx]\n        y_train = y.iloc[train_idx]\n        \n        X_val = X.iloc[val_idx]\n        y_val = y.iloc[val_idx]\n        \n        val_preds = np.zeros((len(X_val), 4))\n        val_score = 0\n        \n        for i in range(4):\n            model = clone(estimator).fit(X_train, y_train.iloc[:, i])\n            val_preds[:, i] = model.predict(X_val)\n            val_score += accuracy_score(y_val.iloc[:, i], val_preds[:, i]) / 4\n            if show_importance:\n                feature_importances_table['value'] += permutation_importance(model, X_val, y_val.iloc[:, i], random_state = seed, scoring = 'accuracy', n_repeats = 5).importances_mean / 4 / cv.get_n_splits()\n            \n        val_scores.append(val_score)\n    \n    print(f'Val Score: {np.mean(val_scores):.5f} ± {np.std(val_scores):.5f} | {label}')\n    \n    if show_importance:\n        plt.figure(figsize = (20, 50))\n        plt.title('Feature Importances', weight = 'bold', size = 25)\n        sns.barplot(feature_importances_table.sort_values('value', ascending = False).T, orient = 'h', palette = 'viridis')\n        plt.show()\n    \n    return val_scores, feature_importances_table","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:48.106164Z","iopub.execute_input":"2024-03-14T20:29:48.106421Z","iopub.status.idle":"2024-03-14T20:29:48.120913Z","shell.execute_reply.started":"2024-03-14T20:29:48.106399Z","shell.execute_reply":"2024-03-14T20:29:48.120088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cross_val_score_proba(estimator, col, cv = kf, label = ''):\n    \n    val_scores = []\n    val_predictions = np.zeros((len(X), y.loc[:, col].nunique()))\n    test_predictions = np.zeros((len(test_pd), y.loc[:, col].nunique()))\n    \n    for fold, (train_idx, val_idx) in enumerate(cv.split(X, y)):\n        \n        X_train = X.iloc[train_idx].reset_index(drop = True)\n        y_train = y.iloc[train_idx].reset_index(drop = True)\n        \n        X_val = X.iloc[val_idx].reset_index(drop = True)\n        y_val = y.iloc[val_idx].reset_index(drop = True)\n        \n        model = clone(estimator).fit(X_train, y_train[col])\n        val_preds = model.predict_proba(X_val)\n        val_score = accuracy_score(y_val[col], val_preds.argmax(axis = 1))\n            \n        val_scores.append(val_score)\n        val_predictions[val_idx] = val_preds\n        test_predictions += model.predict_proba(test_pd) / cv.get_n_splits()\n        \n        #print(f'Fold Score: {val_score:.5f}')\n    \n    print(f'Val Score: {np.mean(val_scores):.5f} ± {np.std(val_scores):.5f} | {label}')\n    \n    return val_scores, val_predictions, test_predictions","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:48.121872Z","iopub.execute_input":"2024-03-14T20:29:48.122101Z","iopub.status.idle":"2024-03-14T20:29:48.136282Z","shell.execute_reply.started":"2024-03-14T20:29:48.122080Z","shell.execute_reply":"2024-03-14T20:29:48.135438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_list = {col : [] for col in list(y)}\npredict_list = {col : [] for col in list(y)}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:48.137505Z","iopub.execute_input":"2024-03-14T20:29:48.137803Z","iopub.status.idle":"2024-03-14T20:29:48.150534Z","shell.execute_reply.started":"2024-03-14T20:29:48.137775Z","shell.execute_reply":"2024-03-14T20:29:48.149796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Logistic Regression","metadata":{}},{"cell_type":"code","source":"log_models = {}\n\nfor col in list(y):\n    log_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        StandardScaler(),\n        PowerTransformer(),\n        LogisticRegression(random_state = seed, max_iter = 10000000)\n    )\n    _, oof_preds, _ = cross_val_score_proba(log_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:29:48.151627Z","iopub.execute_input":"2024-03-14T20:29:48.151914Z","iopub.status.idle":"2024-03-14T20:30:19.866848Z","shell.execute_reply.started":"2024-03-14T20:29:48.151892Z","shell.execute_reply":"2024-03-14T20:30:19.861570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SVM","metadata":{}},{"cell_type":"code","source":"svm_models = {}\n\nfor col in list(y):\n    svm_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        StandardScaler(),\n        PowerTransformer(),\n        SVC(random_state = seed, probability = True, max_iter = 1000000)\n    )\n    _, oof_preds, _ = cross_val_score_proba(svm_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:30:19.879776Z","iopub.execute_input":"2024-03-14T20:30:19.880626Z","iopub.status.idle":"2024-03-14T20:34:15.207743Z","shell.execute_reply.started":"2024-03-14T20:30:19.880579Z","shell.execute_reply":"2024-03-14T20:34:15.206813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM","metadata":{}},{"cell_type":"code","source":"lgb_models = {}\n\nfor col in list(y):\n    lgb_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        LGBMClassifier(random_state = seed)\n    )\n    _, oof_preds, _ = cross_val_score_proba(lgb_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:34:15.209082Z","iopub.execute_input":"2024-03-14T20:34:15.209378Z","iopub.status.idle":"2024-03-14T20:35:11.187236Z","shell.execute_reply.started":"2024-03-14T20:34:15.209352Z","shell.execute_reply":"2024-03-14T20:35:11.186300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"xgb_models = {}\n\nfor col in list(y):\n    xgb_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        XGBClassifier(random_state = seed, tree_method = 'hist')\n    )\n    _, oof_preds, _ = cross_val_score_proba(xgb_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:35:11.188724Z","iopub.execute_input":"2024-03-14T20:35:11.189047Z","iopub.status.idle":"2024-03-14T20:36:17.809172Z","shell.execute_reply.started":"2024-03-14T20:35:11.189020Z","shell.execute_reply":"2024-03-14T20:36:17.808368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble NN","metadata":{}},{"cell_type":"code","source":"def cross_val_score_nn(estimator, col, cv = kf, label = ''):\n    \n    X_oof = oof_list[col]\n        \n    model = clone(estimator).fit(X_oof, y[col])\n    val_preds = model.predict_proba(X_oof)\n    val_score = accuracy_score(y[col], val_preds.argmax(axis = 1))\n    \n    print(f'Val Score: {val_score:.5f} | {label}')\n    \n    return val_score","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:36:17.810329Z","iopub.execute_input":"2024-03-14T20:36:17.810808Z","iopub.status.idle":"2024-03-14T20:36:17.816456Z","shell.execute_reply.started":"2024-03-14T20:36:17.810780Z","shell.execute_reply":"2024-03-14T20:36:17.815261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TensorFlower(BaseEstimator, ClassifierMixin):\n    \n    def fit(self, x, y):\n        \n        tf.keras.utils.set_random_seed(seed)\n        tf.config.experimental.enable_op_determinism()\n        \n        inputs = []\n        norm_inputs = []\n        \n        multiply = []\n        \n        for i in range(len(x)):\n            inputs.append(tf.keras.Input((len(np.unique(y)), )))\n            norm_inputs.append(tf.keras.layers.BatchNormalization()(inputs[i]))\n            \n        concat = tf.keras.layers.Concatenate()(norm_inputs)\n        \n        for i in range(len(x)):\n            dense = tf.keras.layers.Dense(len(np.unique(y)))(concat)\n            norm = tf.keras.layers.BatchNormalization()(dense)\n            act = tf.keras.activations.softmax(norm)\n            multiply.append(tf.keras.layers.Multiply()([inputs[i], act]))\n            \n        outputs = tf.keras.layers.Add()(multiply)\n        \n        self.model = tf.keras.Model(inputs, outputs)\n        self.model.compile(loss = 'sparse_categorical_crossentropy', optimizer = tf.keras.optimizers.AdamW(1e-4))\n        \n        self.model.fit(x, y, epochs = 50, verbose = 0)\n        \n        return self\n    \n    def predict_proba(self, x):\n        \n        return self.model.predict(x, verbose = 0)\n    \n    def predict(self, x):\n        \n        return np.argmax(self.predict_proba(x), axis = 1)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:36:17.818033Z","iopub.execute_input":"2024-03-14T20:36:17.818449Z","iopub.status.idle":"2024-03-14T20:36:17.828409Z","shell.execute_reply.started":"2024-03-14T20:36:17.818425Z","shell.execute_reply":"2024-03-14T20:36:17.827798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in list(y):\n    cross_val_score_nn(TensorFlower(), col)\n    \nprint('\\n')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:36:17.829243Z","iopub.execute_input":"2024-03-14T20:36:17.829815Z","iopub.status.idle":"2024-03-14T20:38:58.833256Z","shell.execute_reply.started":"2024-03-14T20:36:17.829771Z","shell.execute_reply":"2024-03-14T20:38:58.832122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and Submission","metadata":{}},{"cell_type":"code","source":"for col in list(y):\n    predict_list[col].append(log_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(svm_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(lgb_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(xgb_models[col].fit(X, y[col]).predict_proba(test_pd))","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:38:58.834857Z","iopub.execute_input":"2024-03-14T20:38:58.835571Z","iopub.status.idle":"2024-03-14T20:40:40.633417Z","shell.execute_reply.started":"2024-03-14T20:38:58.835530Z","shell.execute_reply":"2024-03-14T20:40:40.632550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in list(y):\n    ensembler = TensorFlower().fit(oof_list[col], y[col])\n    submission[col] = ensembler.predict(predict_list[col])\n    submission[col] = le[col].inverse_transform(submission[col].astype(np.int8))\n\nsubmission[['id_unique'] + list(y)].to_csv('cool_city.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T20:40:40.634813Z","iopub.execute_input":"2024-03-14T20:40:40.635628Z","iopub.status.idle":"2024-03-14T20:43:08.804602Z","shell.execute_reply.started":"2024-03-14T20:40:40.635596Z","shell.execute_reply":"2024-03-14T20:43:08.803806Z"},"trusted":true},"execution_count":null,"outputs":[]}]}