{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":61145,"databundleVersionId":7047669,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Applications","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"This notebook has to be created through the competition","metadata":{}},{"cell_type":"markdown","source":"## Table of Contents\n* [Read data](#Read-data)\n* [Data pre-processing](#Data-pre-processing)\n* [Map of all weather stations that measure air temperature](#Map-of-all-weather-stations-that-measure-air-temperature)\n* [What's next?](#What's-next)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport math","metadata":{"execution":{"iopub.status.busy":"2024-03-14T14:30:41.309972Z","iopub.execute_input":"2024-03-14T14:30:41.311100Z","iopub.status.idle":"2024-03-14T14:30:45.348222Z","shell.execute_reply.started":"2024-03-14T14:30:41.311062Z","shell.execute_reply":"2024-03-14T14:30:45.347173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], \n                       keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                       date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0],\n                          keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], \n                          date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:33:14.140192Z","iopub.execute_input":"2024-03-14T03:33:14.140913Z","iopub.status.idle":"2024-03-14T03:35:14.119287Z","shell.execute_reply.started":"2024-03-14T03:33:14.140882Z","shell.execute_reply":"2024-03-14T03:35:14.118162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:35:14.167543Z","iopub.execute_input":"2024-03-14T03:35:14.168482Z","iopub.status.idle":"2024-03-14T03:35:14.184252Z","shell.execute_reply.started":"2024-03-14T03:35:14.168451Z","shell.execute_reply":"2024-03-14T03:35:14.183112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data pre-processing\n\n### Check dates\n\nWhen checking the last date of different kinds of dataframes it becomes clear that the watch survey responses only last until Jan 21, 2023, while the weather data lasts until July 3, 2023.","metadata":{}},{"cell_type":"code","source":"print(df_train.index.max())\nprint(df_solution.index.max())\nprint(df_rain.index.max())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:35:43.015017Z","iopub.execute_input":"2024-03-14T03:35:43.015431Z","iopub.status.idle":"2024-03-14T03:35:43.033660Z","shell.execute_reply.started":"2024-03-14T03:35:43.015401Z","shell.execute_reply":"2024-03-14T03:35:43.032602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't really need the weather data after Jan 21, 2023, so we might just as well get rid of it.","metadata":{}},{"cell_type":"code","source":"df_rain = df_rain[df_rain.index<'2023-01-22']\ndf_temperature = df_temperature[df_temperature.index<'2023-01-22']\ndf_humidity = df_humidity[df_humidity.index<'2023-01-22']\ndf_wind_speed = df_wind_speed[df_wind_speed.index<'2023-01-22']\ndf_wind_direction = df_wind_direction[df_wind_direction.index<'2023-01-22']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:38:39.705244Z","iopub.execute_input":"2024-03-14T03:38:39.705586Z","iopub.status.idle":"2024-03-14T03:38:39.731663Z","shell.execute_reply.started":"2024-03-14T03:38:39.705557Z","shell.execute_reply":"2024-03-14T03:38:39.730700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check weather data","metadata":{}},{"cell_type":"code","source":"# Show new shortened air temperature dataframe\ndf_temperature.describe()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:38:45.395079Z","iopub.execute_input":"2024-03-14T03:38:45.395423Z","iopub.status.idle":"2024-03-14T03:38:45.530640Z","shell.execute_reply.started":"2024-03-14T03:38:45.395398Z","shell.execute_reply":"2024-03-14T03:38:45.529634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like there is no temperature data available in before Jan 21, 2023 for weather station S117. So, we might just as well remove that column","metadata":{}},{"cell_type":"code","source":"df_temperature = df_temperature.drop(columns=['S117'])\n# Get dimensions of the air temperature data\ndf_temperature.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:38:57.731535Z","iopub.execute_input":"2024-03-14T03:38:57.732222Z","iopub.status.idle":"2024-03-14T03:38:57.744667Z","shell.execute_reply.started":"2024-03-14T03:38:57.732181Z","shell.execute_reply":"2024-03-14T03:38:57.743755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The weather dataframes are rather large. This can make the plotting of the data slow and the output images large. To make it a bit lighter, we are going to resample the data hourly.","metadata":{"execution":{"iopub.status.busy":"2024-03-12T21:09:04.412698Z","iopub.execute_input":"2024-03-12T21:09:04.413286Z","iopub.status.idle":"2024-03-12T21:09:04.419237Z","shell.execute_reply.started":"2024-03-12T21:09:04.413253Z","shell.execute_reply":"2024-03-12T21:09:04.41811Z"}}},{"cell_type":"code","source":"# Resample weather data\ndf_temperature_average = df_temperature.resample('60min').mean()\ndf_rain_sum = df_rain.resample('60min').sum() # note: for rain the data is aggregated with a sum instead of the mean\ndf_humidity_average = df_humidity.resample('60min').mean()\ndf_wind_speed_average = df_wind_speed.resample('60min').mean()\ndf_wind_direction_average = df_wind_direction.resample('60min').mean()\n\n# Check size of resulting dataframe\ndf_temperature_average.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:39:01.082900Z","iopub.execute_input":"2024-03-14T03:39:01.083279Z","iopub.status.idle":"2024-03-14T03:39:01.173538Z","shell.execute_reply.started":"2024-03-14T03:39:01.083249Z","shell.execute_reply":"2024-03-14T03:39:01.172523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Map watch survey questions to question ID\n","metadata":{}},{"cell_type":"code","source":"ws_questions = {\"q_noise_nearby\": \"Noise distractions nearby? (without earphones)\",\n                \"q_noise_kind\": \"What kind of noise?\",\n                \"q_earphones\": \"Wearing earphones?\",\n                \"q_thermal_preference\": \"Thermally, what do you prefer now?\",\n                \"q_location\": \"Where are you?\",\n                \"q_location_office\": \"What kind of office?\",\n                \"q_location_transport\": \"What kind of transport?\",\n                \"q_alone_group\": \"Alone or in a group?\",\n                \"q_activity_category_alone\": \"Category of activity? (alone)\",\n                \"q_activity_category_group\": \"Category of activity? (group)\"}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:39:30.804145Z","iopub.execute_input":"2024-03-14T03:39:30.804495Z","iopub.status.idle":"2024-03-14T03:39:30.809996Z","shell.execute_reply.started":"2024-03-14T03:39:30.804466Z","shell.execute_reply":"2024-03-14T03:39:30.809074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot weather data\n\n### Plot weather weather data time-series¶\n\nBelow we are going to plot the air temperature data from all weather stations. I used the Matplotlib method plt.plot() instead of sns.lineplot(). The reason is that plt.plot() shows the gaps in the data, while sns.lineplot() draws straight lines accross the gaps.\n","metadata":{}},{"cell_type":"code","source":"cols_num = 1\nrows_num = len(df_temperature_average.columns)\ncol_current = 0\nrow_current = 0 \nfig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\nplt.subplots_adjust(hspace = 0.5)\n\nfor i, col in enumerate(df_temperature_average.columns):\n    row_current = i\n    axs[row_current].plot(df_temperature_average.index, df_temperature_average[col])\n    station_name = df_stations[df_stations['id']==col]['name'].values[0]\n    axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n    axs[row_current].set_ylabel('Air Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:39:36.357619Z","iopub.execute_input":"2024-03-14T03:39:36.358565Z","iopub.status.idle":"2024-03-14T03:39:42.159113Z","shell.execute_reply.started":"2024-03-14T03:39:36.358524Z","shell.execute_reply":"2024-03-14T03:39:42.156012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since, we are going to plot more weather data, we might just as well wrap the above code for plotting in to a function.","metadata":{}},{"cell_type":"code","source":"# Function that plots weather data from all weather stations\ndef plot_weather(df, ylabel):\n    cols_num = 1\n    rows_num = len(df.columns)\n    col_current = 0\n    row_current = 0 \n    fig, axs = plt.subplots(rows_num, cols_num, figsize=(20,3*rows_num))\n    plt.subplots_adjust(hspace = 0.5)\n\n    for i, col in enumerate(df.columns):\n        row_current = i\n        axs[row_current].plot(df.index, df[col])\n        station_name = df_stations[df_stations['id']==col]['name'].values[0]\n        axs[row_current].set_title(f'Weather Station {col}: {station_name}')\n        axs[row_current].set_ylabel(ylabel)\n    \n    return fig, axs","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:40:08.666728Z","iopub.execute_input":"2024-03-14T03:40:08.667398Z","iopub.status.idle":"2024-03-14T03:40:08.674534Z","shell.execute_reply.started":"2024-03-14T03:40:08.667367Z","shell.execute_reply":"2024-03-14T03:40:08.673436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Temperature","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_temperature_average, 'Temperature [°C]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:40:22.860352Z","iopub.execute_input":"2024-03-14T03:40:22.860760Z","iopub.status.idle":"2024-03-14T03:40:28.448249Z","shell.execute_reply.started":"2024-03-14T03:40:22.860731Z","shell.execute_reply":"2024-03-14T03:40:28.447321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Rainfall","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_rain_sum, 'Rainfall [mm/h]')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:40:59.555678Z","iopub.execute_input":"2024-03-14T03:40:59.556352Z","iopub.status.idle":"2024-03-14T03:41:22.456392Z","shell.execute_reply.started":"2024-03-14T03:40:59.556316Z","shell.execute_reply":"2024-03-14T03:41:22.453872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relative humidity","metadata":{}},{"cell_type":"code","source":"fig, axs = plot_weather(df_humidity_average, 'Relative humidity [%]')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-12T21:12:03.695307Z","iopub.execute_input":"2024-03-12T21:12:03.695626Z","iopub.status.idle":"2024-03-12T21:12:09.247303Z","shell.execute_reply.started":"2024-03-12T21:12:03.6956Z","shell.execute_reply":"2024-03-12T21:12:09.246006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Map of all weather stations that measure air temperature","metadata":{}},{"cell_type":"code","source":"# Have a look at the weather station meta data\ndf_stations.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:42:14.485921Z","iopub.execute_input":"2024-03-14T03:42:14.486271Z","iopub.status.idle":"2024-03-14T03:42:14.497822Z","shell.execute_reply.started":"2024-03-14T03:42:14.486242Z","shell.execute_reply":"2024-03-14T03:42:14.496859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_stations.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:42:17.358870Z","iopub.execute_input":"2024-03-14T03:42:17.359239Z","iopub.status.idle":"2024-03-14T03:42:17.365123Z","shell.execute_reply.started":"2024-03-14T03:42:17.359207Z","shell.execute_reply":"2024-03-14T03:42:17.364231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a look at the air temperature data\ndf_temperature.columns","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:42:19.867633Z","iopub.execute_input":"2024-03-14T03:42:19.868226Z","iopub.status.idle":"2024-03-14T03:42:19.873959Z","shell.execute_reply.started":"2024-03-14T03:42:19.868199Z","shell.execute_reply":"2024-03-14T03:42:19.873148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, the list of all weather stations is much longer then the list with weather stations that can measure air temperature. Hence, we need to pre-process the data a bit and only pick the weather stations that can measure temperature data.","metadata":{}},{"cell_type":"code","source":"# Initialize a new dataframe for creating the map\ndf_map = pd.DataFrame(columns=['latitude', 'longitude', 'text'])\n\n\n# Go through each column of the temperature dataframe\nfor col in df_temperature.columns:\n    # Get the weather station name based on the station id\n    station_name = df_stations.loc[df_stations.id==col].name.values[0]\n    \n    # Create a new row, with latitude, longitude, hover text, and marker color as columns\n    df_row = pd.DataFrame([{'latitude': df_stations.loc[df_stations.id==col].latitude.values[0], \n                            'longitude': df_stations.loc[df_stations.id==col].longitude.values[0], \n                            'text': f'Station ID: {col} <br>Station name: {station_name} <br>Measurement: Rain',\n                            'color': 'rgb(0, 128, 255)'}])\n    \n    # Append new row to map dataframe\n    df_map = pd.concat([df_map, df_row])\n    \ndf_map.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:42:23.619438Z","iopub.execute_input":"2024-03-14T03:42:23.619799Z","iopub.status.idle":"2024-03-14T03:42:23.678676Z","shell.execute_reply.started":"2024-03-14T03:42:23.619753Z","shell.execute_reply":"2024-03-14T03:42:23.677803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pre-processing for the maps is done. Now, we can go and actually create the map chart.","metadata":{}},{"cell_type":"code","source":"# Create trace and add markers to map\ntrace = go.Scattermapbox(lat=df_map.latitude, \n                         lon=df_map.longitude,\n                         mode='markers',\n                         marker=dict(size=10,\n                                     color=df_map['color'],\n                                     opacity=0.8),\n                         hovertemplate = \"\",\n                         text=df_map['text'])\n\n# Define the layout for the map\nlayout = go.Layout(\n    margin={\"l\": 0, \"r\": 0, \"t\": 0, \"b\": 0},\n    mapbox=dict(\n        style='carto-positron', # Alternative: 'open-street-map',\n        center=dict(lat=df_map.latitude.median(),\n                    lon=df_map.longitude.median()),\n        zoom=10,\n        ),\n    height=700, width=900)\n\n# Create figure\nfig = go.Figure(data=[trace], layout=layout)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T03:42:28.548927Z","iopub.execute_input":"2024-03-14T03:42:28.549326Z","iopub.status.idle":"2024-03-14T03:42:29.136913Z","shell.execute_reply.started":"2024-03-14T03:42:28.549294Z","shell.execute_reply":"2024-03-14T03:42:29.136062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note, you can hovering the cursor over the markers will reveal some meta information about the weather station.","metadata":{}},{"cell_type":"markdown","source":"## What's next?","metadata":{}},{"cell_type":"markdown","source":"How do I continue this? Where to start?!\n\nGO TO THE COMPETITION SITE!\n\n[https://www.kaggle.com/competitions/cool-quiet-city-competition](https://www.kaggle.com/competitions/cool-quiet-city-competition)","metadata":{}},{"cell_type":"markdown","source":"### Here is my contribution to the competition I signed up for.\nRemenber that, this competition's goal is to use contextual data to predict noise distraction and thermal preference across a diversity of indoor and outdoor spaces.","metadata":{}},{"cell_type":"markdown","source":"# I) Loading Libraries and Datasets","metadata":{}},{"cell_type":"code","source":"# Install the iterative-stratification package using pip\n!pip install -q iterative-stratification","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:53:12.758229Z","iopub.execute_input":"2024-03-14T17:53:12.758894Z","iopub.status.idle":"2024-03-14T17:53:24.763925Z","shell.execute_reply.started":"2024-03-14T17:53:12.758861Z","shell.execute_reply":"2024-03-14T17:53:24.762602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np  # For numerical operations\nimport pandas as pd  # For data manipulation and analysis\nimport polars as pl  # For fast data manipulation\nimport matplotlib.pyplot as plt  # For plotting\nimport seaborn as sns  # For enhanced plotting\nimport tensorflow as tf  # For deep learning\nimport os  # For operating system operations\nimport optuna  # For hyperparameter optimization\nimport holidays  # For handling holidays\nimport copy  # For deep copying objects\n\nfrom category_encoders import OneHotEncoder, MEstimateEncoder, CatBoostEncoder, BinaryEncoder  # For categorical encoding\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold, RepeatedMultilabelStratifiedKFold  # For stratified KFold\nfrom sklearn.cluster import KMeans  # For clustering\nfrom sklearn.compose import ColumnTransformer  # For column transformations\nfrom sklearn.model_selection import StratifiedKFold, RepeatedStratifiedKFold, TimeSeriesSplit  # For cross-validation\nfrom sklearn.decomposition import PCA  # For principal component analysis\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier  # For ensemble models\nfrom sklearn.ensemble import GradientBoostingClassifier, HistGradientBoostingClassifier  # For gradient boosting\nfrom sklearn.ensemble import VotingClassifier  # For ensemble voting\nfrom sklearn.impute import KNNImputer, SimpleImputer  # For imputation\nfrom sklearn.metrics import accuracy_score, roc_auc_score, roc_curve, make_scorer  # For model evaluation\nfrom sklearn.metrics.pairwise import euclidean_distances  # For pairwise distances\nfrom sklearn.naive_bayes import GaussianNB, BernoulliNB  # For Naive Bayes models\nfrom sklearn.pipeline import make_pipeline, Pipeline  # For creating pipelines\nfrom sklearn.base import BaseEstimator, TransformerMixin, ClassifierMixin, clone  # For base classes\nfrom sklearn.multioutput import ClassifierChain  # For multi-output classification\nfrom sklearn.preprocessing import FunctionTransformer, StandardScaler, PowerTransformer, LabelEncoder  # For preprocessing\nfrom sklearn.linear_model import LogisticRegression, RidgeClassifier  # For linear models\nfrom sklearn.svm import SVC  # For support vector machines\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis  # For LDA\nfrom sklearn.neighbors import KNeighborsClassifier  # For KNN\nfrom sklearn.inspection import permutation_importance  # For feature importance\nfrom scipy.cluster.hierarchy import dendrogram, linkage  # For hierarchical clustering\nfrom scipy.spatial.distance import squareform  # For distance calculations\nfrom xgboost import XGBClassifier, XGBRFClassifier  # For XGBoost models\nfrom lightgbm import LGBMClassifier  # For LightGBM models\nfrom catboost import CatBoostClassifier  # For CatBoost models\nfrom sklearn.feature_selection import SequentialFeatureSelector  # For feature selection\n\nsns.set_theme(style='white', palette='viridis')  # Set seaborn theme and palette\npal = sns.color_palette('viridis')  # Define color palette\n\npd.set_option('display.max_rows', 100)  # Set maximum rows displayed in pandas\n","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:53:29.812603Z","iopub.execute_input":"2024-03-14T17:53:29.813506Z","iopub.status.idle":"2024-03-14T17:53:55.166810Z","shell.execute_reply.started":"2024-03-14T17:53:29.813474Z","shell.execute_reply":"2024-03-14T17:53:55.165870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We define three different preprocessors for transforming columns in a Polars DataFrame, these preprocessors are designed to be used with the Polars DataFrame API for data manipulation and transformation.","metadata":{}},{"cell_type":"code","source":"preprocessor = [\n    pl.col('dT').cast(pl.Float32),\n    \n    pl.col('ts_oxygen_saturation').cast(pl.Float32),\n    pl.col('ts_resting_heart_rate').cast(pl.Float32),\n    \n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('time_year').cast(pl.UInt16),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('time_month').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('time_week').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('time_weekday').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('time_day_of_year').cast(pl.UInt16),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('time_day').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('time_hour').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('time_minute').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('time_second').cast(pl.UInt8),\n    pl.col('time').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('time_micro').cast(pl.UInt32),\n    \n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('loc_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('loc_month').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('loc_week').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('loc_weekday').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('loc_day_of_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('loc_day').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('loc_hour').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('loc_minute').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('loc_second').cast(pl.UInt8),\n    pl.col('ws_timestamp_location').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('loc_micro').cast(pl.UInt32),\n    \n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z'),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.year().alias('start_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.month().alias('start_month').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.week().alias('start_week').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.weekday().alias('start_weekday').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.ordinal_day().alias('start_day_of_year').cast(pl.UInt16),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.day().alias('start_day').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.hour().alias('start_hour').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.minute().alias('start_minute').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.second().alias('start_second').cast(pl.UInt8),\n    pl.col('ws_timestamp_start').str.to_datetime('%Y-%m-%d %H:%M:%S%.6f%Z').dt.microsecond().alias('start_micro').cast(pl.UInt32)\n]\n\npreprocessor_2 = [\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z'),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.year().alias('year').cast(pl.UInt16),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.month().alias('month').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.day().alias('day').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.hour().alias('hour').cast(pl.UInt8),\n    pl.col('').str.to_datetime('%Y-%m-%d %H:%M:%S%Z').dt.minute().alias('minute').cast(pl.UInt8),\n    \n    pl.col('value').cast(pl.Float32)\n]\n\nduration_calculator = [\n    (pl.duration(days = pl.col('time_day_of_year'), hours = pl.col('time_hour'), minutes = pl.col('time_minute'), seconds = pl.col('time_second'), microseconds = pl.col('time_micro')) -\n    pl.duration(days = pl.col('start_day_of_year'), hours = pl.col('start_hour'), minutes = pl.col('start_minute'), seconds = pl.col('start_second'), microseconds = pl.col('start_micro'))).dt.total_microseconds().alias('time_start_difference'),\n    (pl.duration(days = pl.col('time_day_of_year'), hours = pl.col('time_hour'), minutes = pl.col('time_minute'), seconds = pl.col('time_second'), microseconds = pl.col('time_micro')) -\n    pl.duration(days = pl.col('loc_day_of_year'), hours = pl.col('loc_hour'), minutes = pl.col('loc_minute'), seconds = pl.col('loc_second'), microseconds = pl.col('loc_micro'))).dt.total_microseconds().alias('time_loc_difference'),\n    (pl.duration(days = pl.col('loc_day_of_year'), hours = pl.col('loc_hour'), minutes = pl.col('loc_minute'), seconds = pl.col('loc_second'), microseconds = pl.col('loc_micro')) -\n     pl.duration(days = pl.col('start_day_of_year'), hours = pl.col('start_hour'), minutes = pl.col('start_minute'), seconds = pl.col('start_second'), microseconds = pl.col('start_micro'))).dt.total_microseconds().alias('start_loc_difference')\n]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:54:15.712005Z","iopub.execute_input":"2024-03-14T17:54:15.712388Z","iopub.status.idle":"2024-03-14T17:54:15.744966Z","shell.execute_reply.started":"2024-03-14T17:54:15.712359Z","shell.execute_reply":"2024-03-14T17:54:15.743851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv').\\\n        with_columns(preprocessor).with_columns(duration_calculator)\ntest = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv').\\\n       with_columns(preprocessor).with_columns(duration_calculator)\n#This code reads two CSV files into Polars DataFrames, applying the specified preprocessors ","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:54:20.042273Z","iopub.execute_input":"2024-03-14T17:54:20.042973Z","iopub.status.idle":"2024-03-14T17:54:20.137392Z","shell.execute_reply.started":"2024-03-14T17:54:20.042941Z","shell.execute_reply":"2024-03-14T17:54:20.136583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#This code reads several weather-related CSV files into Polars DataFrames, applies similar preprocessing steps to each file\nair_temp = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_air_temp'),\n               pl.col('value').std().alias('std_air_temp'),\n           )\n'''\nair_temp: Reads the 'weather_air-temperature.csv' file,\nmelts the columns into rows, applies the preprocessor_2 \nto transform the columns, groups the data by year, month, day, hour, and minute, \nand calculates the mean and standard deviation of the 'value' column (air temperature) \nfor each group.\n'''\nrainfall = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_rainfall'),\n               pl.col('value').std().alias('std_rainfall'),\n           )\nhumidity = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv').\\\n           melt('').\\\n           with_columns(preprocessor_2).\\\n           group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n           agg(\n               pl.col('value').mean().alias('mean_humidity'),\n               pl.col('value').std().alias('std_humidity'),\n           )\nstations = pl.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv')\nwind_direction = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv').\\\n                 melt('').\\\n                 with_columns(preprocessor_2).\\\n                 group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n                 agg(\n                     pl.col('value').mean().alias('mean_wind_direction'),\n                     pl.col('value').std().alias('std_wind_direction')\n                 )\nwind_speed = pl.scan_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv').\\\n             melt('').\\\n             with_columns(preprocessor_2).\\\n             group_by(['year', 'month', 'day', 'hour', 'minute']).\\\n             agg(\n                 pl.col('value').mean().alias('mean_wind_speed'),\n                 pl.col('value').std().alias('std_wind_speed'),\n             )","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:54:25.653519Z","iopub.execute_input":"2024-03-14T17:54:25.654401Z","iopub.status.idle":"2024-03-14T17:54:25.685836Z","shell.execute_reply.started":"2024-03-14T17:54:25.654369Z","shell.execute_reply":"2024-03-14T17:54:25.684990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# II) Dataset Information\nThis function, desc_dataset_pl, generates a summary description of a Polars DataFrame,\nIt first creates a DataFrame containing information about the columns of the input and then calculates additional columns.","metadata":{}},{"cell_type":"code","source":"def desc_dataset_pl(df):\n    return pl.concat([\n        pl.DataFrame({'key' : df.columns, 'dataset_length' : len(df)}),\n        df.null_count().transpose().rename({'column_0' : 'null'}),\n        df.approx_n_unique().transpose().rename({'column_0' : 'nunique'}),\n        df.mean().transpose().rename({'column_0' : 'mean'}),\n        df.std().transpose().rename({'column_0' : 'std'}),\n        df.median().transpose().rename({'column_0' : 'median'}),\n        df.min().transpose().rename({'column_0' : 'min'}),\n        df.max().transpose().rename({'column_0' : 'max'})\n    ], how = 'horizontal').with_columns(\n        (pl.col('dataset_length') - pl.col('null')).alias('count'),\n        (pl.col('null') / pl.col('dataset_length') * 100).alias('%null'),\n        (pl.col('nunique') / pl.col('dataset_length') * 100).alias('%unique')\n    ).drop('dataset_length').to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:54:30.653936Z","iopub.execute_input":"2024-03-14T17:54:30.654569Z","iopub.status.idle":"2024-03-14T17:54:30.662940Z","shell.execute_reply.started":"2024-03-14T17:54:30.654537Z","shell.execute_reply":"2024-03-14T17:54:30.661992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#desc_dataset_pl(train)\n#desc_dataset_pl(test)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:54:47.785007Z","iopub.execute_input":"2024-03-14T17:54:47.785886Z","iopub.status.idle":"2024-03-14T17:54:47.790181Z","shell.execute_reply.started":"2024-03-14T17:54:47.785852Z","shell.execute_reply":"2024-03-14T17:54:47.789117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TS Features Aggregation","metadata":{}},{"cell_type":"code","source":"df_all = pl.concat([train, test], how = 'vertical')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:54:50.633504Z","iopub.execute_input":"2024-03-14T17:54:50.633871Z","iopub.status.idle":"2024-03-14T17:54:50.641881Z","shell.execute_reply.started":"2024-03-14T17:54:50.633845Z","shell.execute_reply":"2024-03-14T17:54:50.640833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code snippet aggregates time series features from the df_all DataFrame, which is a concatenation of the train and test DataFrames. It calculates the number of surveys completed by each participant per day, per hour, and per minute:","metadata":{}},{"cell_type":"code","source":"survey_counter_day = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_day').cast(pl.UInt16)\n)\nsurvey_counter_hour = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_hour').cast(pl.UInt16)\n)\nsurvey_counter_minute = df_all.group_by(\n    ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute']\n).agg(\n    pl.col('id_participant').count().alias('survey_count_per_minute').cast(pl.UInt8)\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:55:30.760653Z","iopub.execute_input":"2024-03-14T17:55:30.761026Z","iopub.status.idle":"2024-03-14T17:55:30.769034Z","shell.execute_reply.started":"2024-03-14T17:55:30.760999Z","shell.execute_reply":"2024-03-14T17:55:30.767908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code snippet aggregates physiological features from the df_all DataFrame, which contains concatenated data from both the train and test DataFrames. It calculates the mean and standard deviation of various physiological metrics, such as resting heart rate, oxygen saturation, stand time, step count, walking distance, audio exposure environment, and heart rate, grouped by participant ID and different time granularities (year, week, day, or month)","metadata":{}},{"cell_type":"code","source":"rest_heart_rate_agg = df_all.group_by(\n    ['time_year', 'time_week', 'id_participant']\n).agg(\n    pl.col('ts_resting_heart_rate').mean().alias('mean_resting_heart_rate'),\n    pl.col('ts_resting_heart_rate').std().alias('std_resting_heart_rate')\n)\noxygen_saturation_agg = df_all.group_by(\n    ['time_year', 'time_week', 'id_participant']\n).agg(\n    pl.col('ts_oxygen_saturation').mean().alias('mean_oxygen_saturation'),\n    pl.col('ts_oxygen_saturation').std().alias('std_oxygen_saturation')\n)\nstand_time_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_stand_time').mean().alias('mean_stand_time'),\n    pl.col('ts_stand_time').std().alias('std_stand_time')\n)\nstep_count_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_step_count').mean().alias('mean_step_count'),\n    pl.col('ts_step_count').std().alias('std_step_count')\n)\nwalking_distance_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_walking_distance').mean().alias('mean_walking_distance'),\n    pl.col('ts_walking_distance').std().alias('std_walking_distance')\n)\naudio_exposure_environment_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_audio_exposure_environment').mean().alias('mean_audio_exposure_environment'),\n    pl.col('ts_audio_exposure_environment').std().alias('std_audio_exposure_environment')\n)\nheart_rate_agg = df_all.group_by(\n    ['time_year', 'time_month', 'time_day', 'id_participant']\n).agg(\n    pl.col('ts_heart_rate').mean().alias('mean_heart_rate'),\n    pl.col('ts_heart_rate').std().alias('std_heart_rate')\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:55:36.535149Z","iopub.execute_input":"2024-03-14T17:55:36.535845Z","iopub.status.idle":"2024-03-14T17:55:36.548532Z","shell.execute_reply.started":"2024-03-14T17:55:36.535813Z","shell.execute_reply":"2024-03-14T17:55:36.547576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This process enriches the dataset with aggregated features, providing additional insights into participants' physiological patterns over time. Also, this code joins the aggregated features (e.g., mean and standard deviation of physiological metrics) back to the train and test DataFrames based on various time granularities (year, week, day, hour, and minute) and participant ID.","metadata":{}},{"cell_type":"code","source":"train = train.join(\n    rest_heart_rate_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    oxygen_saturation_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    stand_time_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    step_count_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    walking_distance_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    audio_exposure_environment_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    heart_rate_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    survey_counter_day, on = ['id_participant', 'time_year', 'time_month', 'time_day'], how = 'left'\n).join(\n    survey_counter_hour, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour'], how = 'left'\n).join(\n    survey_counter_minute, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'], how = 'left'\n)\ntest = test.join(\n    rest_heart_rate_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    oxygen_saturation_agg, on = ['time_year', 'time_week', 'id_participant'], how = 'left'\n).join(\n    stand_time_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    step_count_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    walking_distance_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    audio_exposure_environment_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    heart_rate_agg, on = ['time_year', 'time_month', 'time_day', 'id_participant'], how = 'left'\n).join(\n    survey_counter_day, on = ['id_participant', 'time_year', 'time_month', 'time_day'], how = 'left'\n).join(\n    survey_counter_hour, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour'], how = 'left'\n).join(\n    survey_counter_minute, on = ['id_participant', 'time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'], how = 'left'\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:55:42.649359Z","iopub.execute_input":"2024-03-14T17:55:42.649823Z","iopub.status.idle":"2024-03-14T17:55:42.675381Z","shell.execute_reply.started":"2024-03-14T17:55:42.649792Z","shell.execute_reply":"2024-03-14T17:55:42.674434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Station and Weather","metadata":{}},{"cell_type":"markdown","source":"This function find_closest_station calculates the distance between a given latitude and longitude (x) and the latitude and longitude of weather stations.","metadata":{}},{"cell_type":"code","source":"def find_closest_station(x):\n    if x[0] == None: return None\n    latitude = float(x[0][:18])\n    longitude = float(x[0][19:])\n    return stations.row(stations.select(\n        distance = (pl.col('latitude') - latitude) ** 2 + (pl.col('longitude') - longitude) ** 2\n    )['distance'].arg_min())[1]","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:56:19.179349Z","iopub.execute_input":"2024-03-14T17:56:19.179764Z","iopub.status.idle":"2024-03-14T17:56:19.186420Z","shell.execute_reply.started":"2024-03-14T17:56:19.179734Z","shell.execute_reply":"2024-03-14T17:56:19.185242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The map_station function takes a DataFrame df and maps each row to the closest weather station. It does this by creating a new DataFrame with the latitude and longitude columns formatted to 18 characters, then applying the find_closest_station function to each row to find the closest station. The function then returns the resulting mapping.","metadata":{}},{"cell_type":"code","source":"def map_station(df):\n    return df.select(pl.col('ws_latitude').cast(pl.Utf8).map_elements(\n        lambda x: x + '0' * (18 - len(x))\n    ) + ' ' + pl.col('ws_longitude').cast(pl.Utf8).map_elements(\n        lambda x: x + '0' * (18 - len(x))\n    )).map_rows(find_closest_station)['map']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:56:33.213586Z","iopub.execute_input":"2024-03-14T17:56:33.214370Z","iopub.status.idle":"2024-03-14T17:56:33.220448Z","shell.execute_reply.started":"2024-03-14T17:56:33.214334Z","shell.execute_reply":"2024-03-14T17:56:33.219395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"(train and test) used for machine learning model training and evaluation. These datasets are being joined with various weather-related dataframes to enrich them with additional features that could potentially improve the model's performance","metadata":{}},{"cell_type":"code","source":"train = train.join(\n    air_temp,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    rainfall,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    humidity,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_direction,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_speed,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute'])\n\ntest = test.join(\n    air_temp,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    rainfall,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    humidity,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_direction,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute']).join(\n    wind_speed,\n    left_on = ['time_year', 'time_month', 'time_day', 'time_hour', 'time_minute'],\n    right_on = ['year', 'month', 'day', 'hour', 'minute'],\n    how = 'left'\n).drop(['year', 'month', 'day', 'hour', 'minute'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:56:37.002319Z","iopub.execute_input":"2024-03-14T17:56:37.002731Z","iopub.status.idle":"2024-03-14T17:56:37.037291Z","shell.execute_reply.started":"2024-03-14T17:56:37.002701Z","shell.execute_reply":"2024-03-14T17:56:37.036083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation\nThis function heatmap generates a heatmap of the correlation matrix for a given dataset. It uses the Pearson correlation method to compute the correlation coefficients between pairs of columns in the dataset.","metadata":{}},{"cell_type":"code","source":"def heatmap(dataset, label = None):\n    corr = dataset.corr(method = 'pearson')\n    plt.figure(figsize = (45, 45), dpi = 300)\n    mask = np.zeros_like(corr)\n    mask[np.triu_indices_from(mask)] = True\n    sns.heatmap(corr, mask = mask, cmap = 'viridis', annot = True, annot_kws = {'size' : 11})\n    plt.title(f'{label} Dataset Correlation Matrix\\n', fontsize = 25, weight = 'bold')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:56:47.995473Z","iopub.execute_input":"2024-03-14T17:56:47.995828Z","iopub.status.idle":"2024-03-14T17:56:48.003507Z","shell.execute_reply.started":"2024-03-14T17:56:47.995799Z","shell.execute_reply":"2024-03-14T17:56:48.002411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation\nPreparing the dataset for a machine learning model, please note that this code snippet is a continuation of the previous code and assumes that the required functions and variables are already defined.","metadata":{}},{"cell_type":"code","source":"X = train.filter(pl.col('q_location').is_not_null()).drop('id_unique', 'time', 'ws_timestamp_location', 'ws_timestamp_start', 'ts_oxygen_saturation', 'ts_resting_heart_rate', 'ts_stand_time', 'ts_step_count', 'ts_walking_distance', 'ts_heart_rate', 'ts_audio_exposure_environment').collect()\nX = X.with_columns(stations = map_station(X)).to_pandas()\ny = X[['q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference']]\nX = X.drop(['q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference'], axis = 1)\n\nsubmission = test.filter(pl.col('q_location').is_not_null()).drop('q_earphones', 'q_noise_kind', 'q_noise_nearby', 'q_thermal_preference', 'time', 'ws_timestamp_location', 'ws_timestamp_start', 'ts_oxygen_saturation', 'ts_resting_heart_rate', 'ts_stand_time', 'ts_step_count', 'ts_walking_distance', 'ts_heart_rate', 'ts_audio_exposure_environment').collect()\nsubmission = submission.with_columns(stations = map_station(submission)).to_pandas()\ntest_pd = submission.drop('id_unique', axis = 1)\ntest_pd.loc[3178, 'time_loc_difference'] += 1e6 * 3600 * 24 * 365\ntest_pd.loc[3178, 'start_loc_difference'] -= 1e6 * 3600 * 24 * 365\n\ncategorical_features = list(X.select_dtypes('object'))\n\nle = {}\n\nfor col in y.columns:\n    le[col] = LabelEncoder().fit(y[col])\n    y[col] = le[col].transform(y[col])\n    \nacc_scorer = make_scorer(accuracy_score)\nseed = 42\ntf.keras.utils.set_random_seed(seed)\n\nkf = MultilabelStratifiedKFold(n_splits = 5, random_state = seed, shuffle = True)\n#kf = TimeSeriesSplit()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:56:51.257429Z","iopub.execute_input":"2024-03-14T17:56:51.257835Z","iopub.status.idle":"2024-03-14T17:57:31.726057Z","shell.execute_reply.started":"2024-03-14T17:56:51.257807Z","shell.execute_reply":"2024-03-14T17:57:31.724738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering\n\nThis NumericalImputer class is a custom transformer that imputes missing values in numerical columns using SimpleImputer.","metadata":{}},{"cell_type":"code","source":"class NumericalImputer(BaseEstimator, TransformerMixin):\n    \n    def fit(self, x, y = None):\n        self.imputer = SimpleImputer().fit(x.select_dtypes(['int', 'float']), y)\n        return self\n    \n    def transform(self, x):\n        x_copy = x.copy()\n        x_copy.loc[:, list(x_copy.select_dtypes(['int', 'float']))] = self.imputer.transform(x_copy.loc[:, list(x_copy.select_dtypes(['int', 'float']))])\n        return x_copy","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:57:57.802943Z","iopub.execute_input":"2024-03-14T17:57:57.803744Z","iopub.status.idle":"2024-03-14T17:57:57.810523Z","shell.execute_reply.started":"2024-03-14T17:57:57.803713Z","shell.execute_reply":"2024-03-14T17:57:57.809612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adversarial Validation\nThis function performs adversarial validation between two datasets, typically a training dataset (dataset_1) and a test dataset (dataset_2). Adversarial validation is a technique used to check if two datasets come from the same distribution, which can be useful in cases where the test set might differ significantly from the training set.","metadata":{}},{"cell_type":"code","source":"def adversarial_validation(dataset_1 = train, dataset_2 = test, label = 'Train-Test'):\n\n    adv_train = dataset_1.copy()\n    adv_test = dataset_2.copy()\n\n    adv_train['is_test'] = 0\n    adv_test['is_test'] = 1\n\n    adv = pd.concat([adv_train, adv_test], ignore_index = True)\n\n    adv_shuffled = adv.sample(frac = 1)\n\n    adv_X = adv_shuffled.drop('is_test', axis = 1)\n    adv_y = adv_shuffled.is_test\n\n    skf = StratifiedKFold(n_splits = 5, random_state = 42, shuffle = True)\n\n    val_scores = []\n    predictions = np.zeros(len(adv))\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(adv_X, adv_y)):\n    \n        adv_lr = make_pipeline(\n            MEstimateEncoder(cols = list(categorical_features)), \n            #FeatureDropper(cols = ['ws_survey_count', 'id_participant']),\n            SimpleImputer(),\n            XGBClassifier(random_state = seed, tree_method = 'hist')\n        )\n        adv_lr.fit(adv_X.iloc[train_idx], adv_y.iloc[train_idx])\n        \n        val_preds = adv_lr.predict_proba(adv_X.iloc[val_idx])[:, 1]\n        predictions[val_idx] = val_preds\n        val_score = roc_auc_score(adv_y.iloc[val_idx], val_preds)\n        val_scores.append(val_score)\n    \n    fpr, tpr, _ = roc_curve(adv['is_test'], predictions)\n    \n    plt.figure(figsize = (10, 10), dpi = 300)\n    sns.lineplot(x=[0, 1], y=[0, 1], linestyle=\"--\", label=\"Indistinguishable Datasets\")\n    sns.lineplot(x=fpr, y=tpr, label=\"Adversarial Validation Classifier\")\n    plt.title(f'{label} Validation = {np.mean(val_scores):.5f}', weight = 'bold', size = 17)\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#adversarial_validation(X, test_pd)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross Validation\nThis function computes the cross-validated score for a given estimator using a specified cross-validation strategy (cv). It also has an option to show feature importances if show_importance is set to True","metadata":{}},{"cell_type":"code","source":"def cross_val_score(estimator, cv = kf, label = '', show_importance = False):\n    \n    val_scores = []\n    \n    feature_importances_table = pd.DataFrame({'value' : 0}, index = list(X.columns))\n    \n    for fold, (train_idx, val_idx) in enumerate(cv.split(X, y)):\n        \n        X_train = X.iloc[train_idx]\n        y_train = y.iloc[train_idx]\n        \n        X_val = X.iloc[val_idx]\n        y_val = y.iloc[val_idx]\n        \n        val_preds = np.zeros((len(X_val), 4))\n        val_score = 0\n        \n        for i in range(4):\n            model = clone(estimator).fit(X_train, y_train.iloc[:, i])\n            val_preds[:, i] = model.predict(X_val)\n            val_score += accuracy_score(y_val.iloc[:, i], val_preds[:, i]) / 4\n            if show_importance:\n                feature_importances_table['value'] += permutation_importance(model, X_val, y_val.iloc[:, i], random_state = seed, scoring = 'accuracy', n_repeats = 5).importances_mean / 4 / cv.get_n_splits()\n            \n        val_scores.append(val_score)\n    \n    print(f'Val Score: {np.mean(val_scores):.5f} ± {np.std(val_scores):.5f} | {label}')\n    \n    if show_importance:\n        plt.figure(figsize = (20, 50))\n        plt.title('Feature Importances', weight = 'bold', size = 25)\n        sns.barplot(feature_importances_table.sort_values('value', ascending = False).T, orient = 'h', palette = 'viridis')\n        plt.show()\n    \n    return val_scores, feature_importances_table","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:58:25.791695Z","iopub.execute_input":"2024-03-14T17:58:25.792083Z","iopub.status.idle":"2024-03-14T17:58:25.804298Z","shell.execute_reply.started":"2024-03-14T17:58:25.792054Z","shell.execute_reply":"2024-03-14T17:58:25.803359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function is similar to the cross_val_score function but is specialized for multi-class classification problems where the target variable has more than two classes.","metadata":{}},{"cell_type":"code","source":"def cross_val_score_proba(estimator, col, cv = kf, label = ''):\n    \n    val_scores = []\n    val_predictions = np.zeros((len(X), y.loc[:, col].nunique()))\n    test_predictions = np.zeros((len(test_pd), y.loc[:, col].nunique()))\n    \n    for fold, (train_idx, val_idx) in enumerate(cv.split(X, y)):\n        \n        X_train = X.iloc[train_idx].reset_index(drop = True)\n        y_train = y.iloc[train_idx].reset_index(drop = True)\n        \n        X_val = X.iloc[val_idx].reset_index(drop = True)\n        y_val = y.iloc[val_idx].reset_index(drop = True)\n        \n        model = clone(estimator).fit(X_train, y_train[col])\n        val_preds = model.predict_proba(X_val)\n        val_score = accuracy_score(y_val[col], val_preds.argmax(axis = 1))\n            \n        val_scores.append(val_score)\n        val_predictions[val_idx] = val_preds\n        test_predictions += model.predict_proba(test_pd) / cv.get_n_splits()\n        \n        #print(f'Fold Score: {val_score:.5f}')\n    \n    print(f'Val Score: {np.mean(val_scores):.5f} ± {np.std(val_scores):.5f} | {label}')\n    \n    return val_scores, val_predictions, test_predictions","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:59:27.563136Z","iopub.execute_input":"2024-03-14T17:59:27.564395Z","iopub.status.idle":"2024-03-14T17:59:27.574427Z","shell.execute_reply.started":"2024-03-14T17:59:27.564357Z","shell.execute_reply":"2024-03-14T17:59:27.573376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are setting up to train models for a multi-label classification problem.","metadata":{}},{"cell_type":"code","source":"oof_list = {col : [] for col in list(y)}\npredict_list = {col : [] for col in list(y)}","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:59:32.307410Z","iopub.execute_input":"2024-03-14T17:59:32.308110Z","iopub.status.idle":"2024-03-14T17:59:32.313290Z","shell.execute_reply.started":"2024-03-14T17:59:32.308076Z","shell.execute_reply":"2024-03-14T17:59:32.312225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Logistic Regression\n\nLet´s use logistic regression models for each column in your target variables, employing techniques like mean encoding, imputation, standard scaling, and power transformation in a pipeline. Then evaluating these models using cross-validation and storing the out-of-fold predictions.","metadata":{}},{"cell_type":"code","source":"log_models = {}\n\nfor col in list(y):\n    log_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        StandardScaler(),\n        PowerTransformer(),\n        LogisticRegression(random_state = seed, max_iter = 10000000)\n    )\n    _, oof_preds, _ = cross_val_score_proba(log_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T17:59:34.824772Z","iopub.execute_input":"2024-03-14T17:59:34.825508Z","iopub.status.idle":"2024-03-14T18:00:09.801810Z","shell.execute_reply.started":"2024-03-14T17:59:34.825473Z","shell.execute_reply":"2024-03-14T18:00:09.800438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SVM\nSupport Vector Machine (SVM) models for each column in your target variables, similar to the logistic regression models. The SVM models are also part of a pipeline that includes mean encoding, imputation, standard scaling, and power transformation.","metadata":{}},{"cell_type":"code","source":"svm_models = {}\n\nfor col in list(y):\n    svm_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        StandardScaler(),\n        PowerTransformer(),\n        SVC(random_state = seed, probability = True, max_iter = 1000000)\n    )\n    _, oof_preds, _ = cross_val_score_proba(svm_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:00:14.247684Z","iopub.execute_input":"2024-03-14T18:00:14.248658Z","iopub.status.idle":"2024-03-14T18:04:18.172859Z","shell.execute_reply.started":"2024-03-14T18:00:14.248625Z","shell.execute_reply":"2024-03-14T18:04:18.171842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM\n\nLightGBM model for each column in target variables, using a pipeline that includes mean encoding and imputation. These models are being evaluated using cross-validation, and you're storing the out-of-fold predictions for further analysis or ensemble methods","metadata":{}},{"cell_type":"code","source":"lgb_models = {}\n\nfor col in list(y):\n    lgb_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        LGBMClassifier(random_state = seed)\n    )\n    _, oof_preds, _ = cross_val_score_proba(lgb_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:04:54.493901Z","iopub.execute_input":"2024-03-14T18:04:54.494318Z","iopub.status.idle":"2024-03-14T18:05:52.292316Z","shell.execute_reply.started":"2024-03-14T18:04:54.494289Z","shell.execute_reply":"2024-03-14T18:05:52.291378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CatBoost\n\nCreating a dictionary of CatBoost models for each column in your target variable y using cross-validation.","metadata":{}},{"cell_type":"code","source":"cb_models = {}\n\nfor col in list(y):\n    cb_models[col] = make_pipeline(\n        NumericalImputer(),\n        CatBoostClassifier(random_state = seed, verbose = 0, cat_features = categorical_features)\n    )\n    _, oof_preds, _ = cross_val_score_proba(cb_models[col], col)\n    oof_list[col].append(oof_preds)\n\nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:08:01.212352Z","iopub.execute_input":"2024-03-14T18:08:01.212767Z","iopub.status.idle":"2024-03-14T18:16:04.624742Z","shell.execute_reply.started":"2024-03-14T18:08:01.212738Z","shell.execute_reply":"2024-03-14T18:16:04.623299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating CatBoost models with the Bayesian bootstrap method using bootstrap_type='Bayesian'.","metadata":{}},{"cell_type":"code","source":"cb_bayes_models = {}\n\nfor col in list(y):\n    cb_bayes_models[col] = make_pipeline(\n        NumericalImputer(),\n        CatBoostClassifier(random_state = seed, verbose = 0, bootstrap_type = 'Bayesian', cat_features = categorical_features)\n    )\n    _, oof_preds, _ = cross_val_score_proba(cb_bayes_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:16:15.565054Z","iopub.execute_input":"2024-03-14T18:16:15.565441Z","iopub.status.idle":"2024-03-14T18:18:21.882228Z","shell.execute_reply.started":"2024-03-14T18:16:15.565412Z","shell.execute_reply":"2024-03-14T18:18:21.880861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating CatBoost models with different bootstrap_type parameters (Bayesian and Bernoulli) for each column in your target variable y using cross-validation.","metadata":{}},{"cell_type":"code","source":"cb_bernoulli_models = {}\n\nfor col in list(y):\n    cb_bernoulli_models[col] = make_pipeline(\n        NumericalImputer(),\n        CatBoostClassifier(random_state = seed, verbose = 0, bootstrap_type = 'Bernoulli', cat_features = categorical_features)\n    )\n    _, oof_preds, _ = cross_val_score_proba(cb_bernoulli_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:18:32.264319Z","iopub.execute_input":"2024-03-14T18:18:32.264975Z","iopub.status.idle":"2024-03-14T18:20:08.043793Z","shell.execute_reply.started":"2024-03-14T18:18:32.264943Z","shell.execute_reply":"2024-03-14T18:20:08.042391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"xgb_models = {}\n\nfor col in list(y):\n    xgb_models[col] = make_pipeline(\n        MEstimateEncoder(cols = categorical_features),\n        SimpleImputer(),\n        XGBClassifier(random_state = seed, tree_method = 'hist')\n    )\n    _, oof_preds, _ = cross_val_score_proba(xgb_models[col], col)\n    oof_list[col].append(oof_preds)\n    \nprint('')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:20:19.993281Z","iopub.execute_input":"2024-03-14T18:20:19.993671Z","iopub.status.idle":"2024-03-14T18:21:28.175744Z","shell.execute_reply.started":"2024-03-14T18:20:19.993642Z","shell.execute_reply":"2024-03-14T18:21:28.174900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble NN\n\nDefining a function cross_val_score_nn to calculate the validation score for a given estimator and column in your target variable y.","metadata":{}},{"cell_type":"code","source":"def cross_val_score_nn(estimator, col, cv = kf, label = ''):\n    \n    X_oof = oof_list[col]\n        \n    model = clone(estimator).fit(X_oof, y[col])\n    val_preds = model.predict_proba(X_oof)\n    val_score = accuracy_score(y[col], val_preds.argmax(axis = 1))\n    \n    print(f'Val Score: {val_score:.5f} | {label}')\n    \n    return val_score","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:22:03.109258Z","iopub.execute_input":"2024-03-14T18:22:03.110237Z","iopub.status.idle":"2024-03-14T18:22:03.115937Z","shell.execute_reply.started":"2024-03-14T18:22:03.110203Z","shell.execute_reply":"2024-03-14T18:22:03.114952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TensorFlower class defines a custom TensorFlow model for classification. This is a neural network architecture with some unique characteristics","metadata":{}},{"cell_type":"code","source":"class TensorFlower(BaseEstimator, ClassifierMixin):\n    \n    def fit(self, x, y):\n        \n        tf.keras.utils.set_random_seed(seed)\n        tf.config.experimental.enable_op_determinism()\n        \n        inputs = []\n        norm_inputs = []\n        \n        multiply = []\n        \n        for i in range(len(x)):\n            inputs.append(tf.keras.Input((len(np.unique(y)), )))\n            norm_inputs.append(tf.keras.layers.BatchNormalization()(inputs[i]))\n            \n        concat = tf.keras.layers.Concatenate()(norm_inputs)\n        \n        for i in range(len(x)):\n            dense = tf.keras.layers.Dense(len(np.unique(y)))(concat)\n            norm = tf.keras.layers.BatchNormalization()(dense)\n            act = tf.keras.activations.softmax(norm)\n            multiply.append(tf.keras.layers.Multiply()([inputs[i], act]))\n            \n        outputs = tf.keras.layers.Add()(multiply)\n        \n        self.model = tf.keras.Model(inputs, outputs)\n        self.model.compile(loss = 'sparse_categorical_crossentropy', optimizer = tf.keras.optimizers.AdamW(1e-4))\n        \n        self.model.fit(x, y, epochs = 50, verbose = 0)\n        \n        return self\n    \n    def predict_proba(self, x):\n        \n        return self.model.predict(x, verbose = 0)\n    \n    def predict(self, x):\n        \n        return np.argmax(self.predict_proba(x), axis = 1)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:22:23.460048Z","iopub.execute_input":"2024-03-14T18:22:23.460672Z","iopub.status.idle":"2024-03-14T18:22:23.471564Z","shell.execute_reply.started":"2024-03-14T18:22:23.460641Z","shell.execute_reply":"2024-03-14T18:22:23.470642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"cross_val_score_nn function evaluates the performance of the TensorFlower function","metadata":{}},{"cell_type":"code","source":"for col in list(y):\n    cross_val_score_nn(TensorFlower(), col)\n    \nprint('\\n')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:22:38.747792Z","iopub.execute_input":"2024-03-14T18:22:38.748495Z","iopub.status.idle":"2024-03-14T18:25:14.817634Z","shell.execute_reply.started":"2024-03-14T18:22:38.748466Z","shell.execute_reply":"2024-03-14T18:25:14.816402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and Submission\n\nLet´s make predictions for each model on the test data test_pd and store the predictions in predict_list.","metadata":{}},{"cell_type":"code","source":"for col in list(y):\n    predict_list[col].append(log_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(svm_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(lgb_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(cb_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(cb_bayes_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(cb_bernoulli_models[col].fit(X, y[col]).predict_proba(test_pd))\n    predict_list[col].append(xgb_models[col].fit(X, y[col]).predict_proba(test_pd))","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:25:38.585970Z","iopub.execute_input":"2024-03-14T18:25:38.586342Z","iopub.status.idle":"2024-03-14T18:31:08.588483Z","shell.execute_reply.started":"2024-03-14T18:25:38.586304Z","shell.execute_reply":"2024-03-14T18:31:08.587116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code snippet demonstrates a workflow for ensembling multiple models and generating predictions for a multi-label classification problem.","metadata":{}},{"cell_type":"code","source":"for col in list(y):\n    ensembler = TensorFlower().fit(oof_list[col], y[col])\n    submission[col] = ensembler.predict(predict_list[col])\n    submission[col] = le[col].inverse_transform(submission[col].astype(np.int8))\n\nsubmission[['id_unique'] + list(y)].to_csv('cool_city.csv', index = False)\n#It creates a submission DataFrame (submission) containing the 'id_unique' column and all columns from y, with the predicted class labels.\n#It saves the submission DataFrame to a CSV file named 'cool_city.csv' for submission or further analysis.","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:31:13.513604Z","iopub.execute_input":"2024-03-14T18:31:13.514378Z","iopub.status.idle":"2024-03-14T18:31:52.486130Z","shell.execute_reply.started":"2024-03-14T18:31:13.514339Z","shell.execute_reply":"2024-03-14T18:31:52.484730Z"},"trusted":true},"execution_count":null,"outputs":[]}]}