{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":61145,"databundleVersionId":7047669,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport pandas as pd \nimport numpy as np\nfrom datetime import datetime, timedelta\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.metrics import confusion_matrix\nfrom math import cos, asin, sqrt, pi\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\n\npd.set_option('display.max_columns', None)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-27T00:03:44.568488Z","iopub.execute_input":"2024-02-27T00:03:44.568884Z","iopub.status.idle":"2024-02-27T00:03:44.576771Z","shell.execute_reply.started":"2024-02-27T00:03:44.568857Z","shell.execute_reply":"2024-02-27T00:03:44.575402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read data\nRead all csv file into dataframes. ","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0], keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])\n","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:03:44.614237Z","iopub.execute_input":"2024-02-27T00:03:44.614625Z","iopub.status.idle":"2024-02-27T00:04:55.202783Z","shell.execute_reply.started":"2024-02-27T00:03:44.614597Z","shell.execute_reply":"2024-02-27T00:04:55.201424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"Create dataframes for the wind direction sine and cosine","metadata":{}},{"cell_type":"code","source":"df_wind_direction_sin = df_wind_direction.copy(deep=True)\ndf_wind_direction_cos = df_wind_direction.copy(deep=True)\nfor s in df_wind_direction.columns:\n    df_wind_direction_sin[s] = np.sin(np.pi/180*df_wind_direction_sin[s])\n    df_wind_direction_cos[s] = np.cos(np.pi/180*df_wind_direction_cos[s])","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:04:55.205349Z","iopub.execute_input":"2024-02-27T00:04:55.205752Z","iopub.status.idle":"2024-02-27T00:04:55.391537Z","shell.execute_reply.started":"2024-02-27T00:04:55.205717Z","shell.execute_reply":"2024-02-27T00:04:55.390113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Merge train and test data in a single dataframe","metadata":{}},{"cell_type":"code","source":"df_train['source']='train'\ndf_solution['source']='solution'\ndf_all = pd.concat([df_train, df_solution])","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:04:55.392908Z","iopub.execute_input":"2024-02-27T00:04:55.393323Z","iopub.status.idle":"2024-02-27T00:04:55.701363Z","shell.execute_reply.started":"2024-02-27T00:04:55.393290Z","shell.execute_reply":"2024-02-27T00:04:55.700450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One hot encoding for categorical features, and dropping the categorical variables","metadata":{}},{"cell_type":"code","source":"df_all['q_Indoor'] = df_all['q_location'].str.contains(\"Indoor\")\ndf_all['q_Class'] = df_all['q_location'].isin(['Indoor - Class'])\ndf_all['q_Home'] = df_all['q_location'].isin(['Indoor - Home'])\ndf_all['q_Office'] = df_all['q_location'].isin(['Indoor - Office'])\ndf_all['q_Indoor_Other'] = df_all['q_location'].isin(['Indoor - Other'])\ndf_all['q_Outdoor'] = df_all['q_location'].isin(['Outdoor'])\ndf_all['q_Transportation'] = df_all['q_location'].isin(['Transportation'])\ndf_all['q_alone_group'] = df_all['q_alone_group'].isin(['Alone'])\ndf_all['q_Bus'] = df_all['q_location_transport'].isin(['Bus'])\ndf_all['q_Car'] = df_all['q_location_transport'].isin(['Car'])\ndf_all['q_Taxi'] = df_all['q_location_transport'].isin(['Taxi'])\ndf_all['q_Train'] = df_all['q_location_transport'].isin(['Train'])\ndf_all['q_Transport_Other'] = df_all['q_location_transport'].isin(['Other'])\ndf_all['q_Focus'] = df_all['q_activity_category_alone'].isin(['Focus'])\ndf_all['q_Leisure'] = df_all['q_activity_category_alone'].isin(['Leisure'])\ndf_all['q_Alone_Activity_Other'] = df_all['q_activity_category_alone'].isin(['Other'])\ndf_all['q_Collaborate'] = df_all['q_activity_category_group'].isin(['Collaborate'])\ndf_all['q_Learn'] = df_all['q_activity_category_group'].isin(['Learn'])\ndf_all['q_Socialize'] = df_all['q_activity_category_group'].isin(['Socialize'])\ndf_all['q_Group_Activity_Other'] = df_all['q_activity_category_group'].isin(['Other'])\ndf_all['q_Conference room'] = df_all['q_location_office'].isin(['Conference room'])\ndf_all['q_Cubicles'] = df_all['q_location_office'].isin(['Cubicles'])\ndf_all['q_Individual'] = df_all['q_location_office'].isin(['Individual'])\ndf_all['q_Large open plan'] = df_all['q_location_office'].isin(['Large open plan'])\ndf_all['q_Not applicable'] = df_all['q_location_office'].isin(['Not applicable'])\ndf_all['q_Small shared'] = df_all['q_location_office'].isin(['Small shared'])\n\ndf_all = df_all.drop(columns=['q_location','q_location_transport','q_activity_category_alone','q_activity_category_group','q_location_office'])","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:04:55.703949Z","iopub.execute_input":"2024-02-27T00:04:55.704300Z","iopub.status.idle":"2024-02-27T00:04:57.130746Z","shell.execute_reply.started":"2024-02-27T00:04:55.704269Z","shell.execute_reply":"2024-02-27T00:04:57.129684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"duration feature represent the log of the time needed to complete the survey, and it's defined as the log of the difference between the time where the survey was submitted and the time where the survey started","metadata":{}},{"cell_type":"code","source":"df_all['duration'] = np.log((df_all.index - df_all['ws_timestamp_start']).dt.total_seconds())","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:04:57.131888Z","iopub.execute_input":"2024-02-27T00:04:57.132717Z","iopub.status.idle":"2024-02-27T00:04:57.186235Z","shell.execute_reply.started":"2024-02-27T00:04:57.132668Z","shell.execute_reply":"2024-02-27T00:04:57.185458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compute day and time features","metadata":{}},{"cell_type":"code","source":"df_all['time_of_day_cos'] = np.cos(np.deg2rad((df_all.index.hour*60 + df_all.index.minute)/4))\ndf_all['time_of_day_sin'] = np.sin(np.deg2rad((df_all.index.hour*60 + df_all.index.minute)/4))\ndf_all['time_of_day'] = (df_all.index.hour*60 + df_all.index.minute)\ndf_all['day_of_week'] = df_all.index.dayofweek\n#df_all = df_all.reset_index()\n#df_all = df_all.sort_values(['id_participant', 'time'])\n#df_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:04:57.187994Z","iopub.execute_input":"2024-02-27T00:04:57.188367Z","iopub.status.idle":"2024-02-27T00:04:57.717436Z","shell.execute_reply.started":"2024-02-27T00:04:57.188333Z","shell.execute_reply":"2024-02-27T00:04:57.716575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compute aggreagates to the features provided by Apple HealthKit","metadata":{}},{"cell_type":"code","source":"df_all['med_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('median')\ndf_all['med_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('median')\ndf_all['med_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('median')\ndf_all['med_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('median')\ndf_all['med_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('median')\ndf_all['med_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('median')\ndf_all['med_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('median')\n\ndf_all['mean_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('mean')\ndf_all['mean_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('mean')\ndf_all['mean_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('mean')\ndf_all['mean_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('mean')\ndf_all['mean_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('mean')\ndf_all['mean_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('mean')\ndf_all['mean_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('mean')\n\ndf_all['std_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('std')\ndf_all['std_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('std')\ndf_all['std_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('std')\ndf_all['std_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('std')\ndf_all['std_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('std')\ndf_all['std_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('std')\ndf_all['std_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('std')","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:04:57.718547Z","iopub.execute_input":"2024-02-27T00:04:57.718802Z","iopub.status.idle":"2024-02-27T00:05:01.251430Z","shell.execute_reply.started":"2024-02-27T00:04:57.718781Z","shell.execute_reply":"2024-02-27T00:05:01.250007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Normalize the features provided by Apple HealthKit","metadata":{}},{"cell_type":"code","source":"df_all['ts_resting_heart_rate'] = (df_all['ts_resting_heart_rate'] - df_all['mean_resting_heart_rate'])/df_all['std_resting_heart_rate']\ndf_all['ts_audio_exposure_environment'] = (df_all['ts_audio_exposure_environment'] - df_all['mean_audio_exposure_environment'])/df_all['std_audio_exposure_environment']\ndf_all['ts_oxygen_saturation'] = (df_all['ts_oxygen_saturation'] - df_all['mean_oxygen_saturation'])/df_all['std_oxygen_saturation']\ndf_all['ts_stand_time'] = (df_all['ts_stand_time'] - df_all['mean_stand_time'])/df_all['std_stand_time']\ndf_all['ts_heart_rate'] = (df_all['ts_heart_rate'] - df_all['mean_heart_rate'])/df_all['std_heart_rate']\ndf_all['ts_step_count'] = (df_all['ts_step_count'] - df_all['mean_step_count'])/df_all['std_step_count']\ndf_all['ts_walking_distance'] = (df_all['ts_walking_distance'] - df_all['mean_walking_distance'])/df_all['std_walking_distance']","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:05:01.252713Z","iopub.execute_input":"2024-02-27T00:05:01.253009Z","iopub.status.idle":"2024-02-27T00:05:01.312171Z","shell.execute_reply.started":"2024-02-27T00:05:01.252983Z","shell.execute_reply":"2024-02-27T00:05:01.311315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Interpolate the features provided by Apple HealthKit\nHealth data, e.g., ts_heart_reate, and the watch survey responses, e.g., q_thermal_preference, do not have coinciding timestamps. Hence, the health data needs to be interpolated at the timestamps of the watch survey responses.\n\nThe interpolated values are stored in columns with the prefix ws_ which stands for 'watch survey', e.g., the interpolated values of ts_heart_rate are stored in ws_heart_rate.","metadata":{}},{"cell_type":"code","source":"# Function to resample health data at the timestamps of watch survey responses.\ndef resample_health(df, col_name, col_name_new):\n    df[col_name_new] = df[col_name].interpolate(method='linear').bfill().ffill().mask(df['ws_survey_count'].isna())\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:05:01.313228Z","iopub.execute_input":"2024-02-27T00:05:01.313487Z","iopub.status.idle":"2024-02-27T00:05:01.320647Z","shell.execute_reply.started":"2024-02-27T00:05:01.313464Z","shell.execute_reply":"2024-02-27T00:05:01.319405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resample normalized health data\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_heart_rate', col_name_new='ws_heart_rate').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_audio_exposure_environment', col_name_new='ws_audio_exposure_environment').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_oxygen_saturation', col_name_new='ws_oxygen_saturation').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_resting_heart_rate', col_name_new='ws_resting_heart_rate').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_stand_time', col_name_new='ws_stand_time').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_step_count', col_name_new='ws_step_count').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_walking_distance', col_name_new='ws_walking_distance').reset_index(level=[0], drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:05:01.324070Z","iopub.execute_input":"2024-02-27T00:05:01.324771Z","iopub.status.idle":"2024-02-27T00:05:23.474093Z","shell.execute_reply.started":"2024-02-27T00:05:01.324737Z","shell.execute_reply":"2024-02-27T00:05:23.472769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Interpolate weather data\nThe weather data is distributed over several dataframes. Each measurement, e.g., rain or air temperature, is in a separate dataframe. The column names are the IDs of the weather stations. There is one dataframe (df_stations), which contains meta information to the weather stations, e.g., coordinates and names.\n\nTo make the weather data usable, we need to find the nearest weather stations for each watch survey response in df_all. Then we can transfer the measurement values with the same timestamp to from the weather data dataframe, e.g., df_rain, to the dataframe with the Cozie data (df_all).","metadata":{}},{"cell_type":"code","source":"# Function to get weather data for a given timestamp and location from the nearest weather station\ndef get_weather_data(df_weather_data, lat, lon, timestamp):    \n    # Get timestamp with no seconds or microseconds\n    timestamp = timestamp.replace(second=0, microsecond=0)\n    \n    # Get weather data for timestamp above\n    weather_data_row = df_weather_data[(df_weather_data.index >= timestamp- timedelta(hours=0, minutes=10)) & (df_weather_data.index <= timestamp + timedelta(hours=0, minutes=10))]\n\n    # If no weather data is available for timestamp above, get weather data for next minute\n    if (weather_data_row.empty):\n        timestamp = timestamp + pd.Timedelta(minutes=1)\n        weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n        # If there is still no weather data available, return NaN\n        if (weather_data_row.empty):\n            timestamp = timestamp - 2 * pd.Timedelta(minutes=1)\n            weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n            if (weather_data_row.empty):\n                return np.NaN\n\n    # Get weather station ids from weather stations with with data\n    station_ids_with_data = weather_data_row.columns[weather_data_row.count() > 0].tolist()\n\n    # Get weather stations with data\n    df_stations2 = df_stations[df_stations.id.isin(station_ids_with_data)]\n\n    # Compute distance to each weather station\n    df_stations2['distance'] = df_stations2.apply(lambda x: distance(x['latitude'], x['longitude'], lat, lon), axis=1)\n\n    # Get nearest weather station\n    station_id_nearest = df_stations2[df_stations2.distance==df_stations2.distance.min()].id.values[0]\n\n    # Retrun NaN for cases where the nearest weather station is more than 10km away\n    if df_stations2.distance.min()>10:\n        return np.NaN\n\n    # Return weather data from nearest weather station\n    return np.nanmean(weather_data_row[station_id_nearest])\n\n# Function to compute distnace on spherical earth\ndef distance(lat1, lon1, lat2, lon2):\n    r = 6371 # km\n    p = pi / 180\n\n    a = 0.5 - cos((lat2-lat1)*p)/2 + cos(lat1*p) * cos(lat2*p) * (1-cos((lon2-lon1)*p))/2\n    return 2 * r * asin(sqrt(a))","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:05:23.475223Z","iopub.execute_input":"2024-02-27T00:05:23.475497Z","iopub.status.idle":"2024-02-27T00:05:23.486130Z","shell.execute_reply.started":"2024-02-27T00:05:23.475473Z","shell.execute_reply":"2024-02-27T00:05:23.485028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply above function to the dataset for all weather data\ndf_all = df_all.reset_index()\ndf_all['ws_rainfall'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_rain, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_air_temperature'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_temperature, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_relative_humidity'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_humidity, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_speed'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_speed, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_sin'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_sin, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_cos'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_cos, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:05:23.487267Z","iopub.execute_input":"2024-02-27T00:05:23.487546Z","iopub.status.idle":"2024-02-27T00:09:13.520029Z","shell.execute_reply.started":"2024-02-27T00:05:23.487503Z","shell.execute_reply":"2024-02-27T00:09:13.518976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Remove unused rows and columns ","metadata":{}},{"cell_type":"code","source":"# Remove all rows that are not a watch surve response\ndf_all = df_all[df_all.ws_survey_count.notna()]\n\n# Remove some timestamp columns\ndf_all = df_all.drop(columns=['ws_timestamp_location', 'ws_timestamp_start'])\n\n# Remove columns of the original health data those are no longer needed since we are going to use the interpolated values\ndf_all = df_all.drop(columns=['ts_heart_rate',\n                              'ts_audio_exposure_environment',\n                              'ts_oxygen_saturation',\n                              'ts_resting_heart_rate',\n                              'ts_stand_time', \n                              'ts_step_count',\n                              'ts_walking_distance'])","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:09:13.521844Z","iopub.execute_input":"2024-02-27T00:09:13.522234Z","iopub.status.idle":"2024-02-27T00:09:13.615001Z","shell.execute_reply.started":"2024-02-27T00:09:13.522201Z","shell.execute_reply":"2024-02-27T00:09:13.613937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering (Part 2)\nThe computation of these features is faster after we have dropped lots of survery rows","metadata":{}},{"cell_type":"markdown","source":"own_earphone and own_noise_cancelling noise cancelling are two features that represent whether a user actually own an earphone or a noise cancelling headset, this information is infered by the user previous answers","metadata":{}},{"cell_type":"code","source":"earphone_dict = df_all.groupby(['id_participant','q_earphones'])['duration'].count().to_dict()\ndf_all['own_earphones'] = 0\ndf_all['own_noise_cancelling'] = 0\nfor index, row in df_all.iterrows():\n    if (row['id_participant'], 'Earphones') in earphone_dict:\n        df_all.at[index, 'own_earphones'] = 1\n    if (row['id_participant'], 'Noise cancelling') in earphone_dict:\n        df_all.at[index, 'own_noise_cancelling'] = 1","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:09:13.616448Z","iopub.execute_input":"2024-02-27T00:09:13.616727Z","iopub.status.idle":"2024-02-27T00:09:16.622816Z","shell.execute_reply.started":"2024-02-27T00:09:13.616706Z","shell.execute_reply":"2024-02-27T00:09:16.621325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"update the features own_earphone and own_noise_cancelling by the fact that nobody uses noise cancelling in class since it defies the purpose of going to class, and the users who haven't used earphones in class, they won't start doing that","metadata":{}},{"cell_type":"code","source":"Use_earphone_in_class = set(df_all[(df_all['q_Class']) & (df_all['q_earphones']=='Earphones')]['id_participant'].unique())\nbeen_to_class = set(df_all[(df_all['q_Class'])&(df_all['source'] == 'train')]['id_participant'].unique())\ndiff = been_to_class.difference(Use_earphone_in_class)\nfor index, row in df_all.iterrows():\n    if row['q_Class']:\n        df_all.at[index, 'own_noise_cancelling'] = 0\n        if row['id_participant'] in diff:\n            df_all.at[index, 'own_earphones'] = 0","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:09:16.624450Z","iopub.execute_input":"2024-02-27T00:09:16.624790Z","iopub.status.idle":"2024-02-27T00:09:17.252368Z","shell.execute_reply.started":"2024-02-27T00:09:16.624765Z","shell.execute_reply":"2024-02-27T00:09:17.251368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Count how many times a location was used in the dataset","metadata":{}},{"cell_type":"code","source":"df_all['location_count'] = 0\nfor index, row in df_all.iterrows():\n    df_all.at[index, 'location_count'] = len(df_all[(df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])])","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:09:17.253820Z","iopub.execute_input":"2024-02-27T00:09:17.254171Z","iopub.status.idle":"2024-02-27T00:09:25.656271Z","shell.execute_reply.started":"2024-02-27T00:09:17.254139Z","shell.execute_reply":"2024-02-27T00:09:25.655569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"calculate the portion of time a user answer a survery from a specific location","metadata":{}},{"cell_type":"code","source":"df_all['location_prob'] = 0\nfor index, row in df_all.iterrows():\n    temp = df_all[(df_all['id_participant']==row['id_participant']) & (df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])]\n    prob = len(temp) / len(df_all[(df_all['id_participant']==row['id_participant'])])\n    df_all.at[index, 'location_prob'] = prob","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:09:25.657609Z","iopub.execute_input":"2024-02-27T00:09:25.657914Z","iopub.status.idle":"2024-02-27T00:09:53.932102Z","shell.execute_reply.started":"2024-02-27T00:09:25.657887Z","shell.execute_reply":"2024-02-27T00:09:53.930813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split data\nWe split the data into the training set and solution set according to the original data","metadata":{}},{"cell_type":"code","source":"# Separte training and solution data again\ndf_train = df_all[df_all.source == 'train'].copy()\ndf_solution = df_all[df_all.source == 'solution'].copy()\n\n# Drop the helper columns ws_surve_count_2 and source again\ndf_train = df_train.drop(columns=['source'])\ndf_solution = df_solution.drop(columns=['source'])","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:09:53.933409Z","iopub.execute_input":"2024-02-27T00:09:53.933778Z","iopub.status.idle":"2024-02-27T00:09:54.030409Z","shell.execute_reply.started":"2024-02-27T00:09:53.933749Z","shell.execute_reply":"2024-02-27T00:09:54.029529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preparation\nFor the machine learning model below, all input and output values need to be numeric. At this point, the watch survey the response columns, e.g., q_noise_nearby, are of type string. Hence, we need convert the values in these columns into numbers.","metadata":{}},{"cell_type":"code","source":"# Find columns with values that are not numeric\ncat_vars = list(df_train.select_dtypes(include=['object']).columns)\n# encoding categorical variables\ndict_cat = {}\n\nfor var in cat_vars:\n    if var == 'id_participant': # no need to encode the participant ID\n        continue\n    # convert categorical variables into 'category' type\n    df_train[var] = df_train[var].astype('category')\n    df_solution[var] = df_solution[var].astype('category')\n    \n    # dictionary to keep track of the variable mapping: {variable-name: {number1: string1, number2: string2, ...}\n    dict_cat[var] = dict(enumerate(df_train[var].cat.categories))\n\n    # encode variables, missing values are encoded as -1\n    df_train[var] = df_train[var].cat.codes\n    df_solution[var] = df_solution[var].cat.codes","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:09:54.031863Z","iopub.execute_input":"2024-02-27T00:09:54.032184Z","iopub.status.idle":"2024-02-27T00:09:54.056432Z","shell.execute_reply.started":"2024-02-27T00:09:54.032156Z","shell.execute_reply":"2024-02-27T00:09:54.055413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict any column\nWe are going to wrap up above code into a function in order to make the prediction of the other columns a bit more convenient. The steps are mostly the same as above and might feel a bit repetitive.","metadata":{}},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier\nfrom sklearn.ensemble import VotingClassifier, StackingClassifier\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_any_column(column_name, df_all, df_solution):\n    # Remove rows where the predicting variable is missing\n    df_all = df_all[df_all[column_name] != -1]\n    \n    # Remove rows where the predicting variable is missing\n    df_all = df_all.drop(columns=['id_participant'])\n    df_solution = df_solution.drop(columns=['id_participant'])\n    \n    # Count classes in predicting variable\n    num_class = df_train[column_name].unique()\n    \n    # Break dataframe into variables and ground truth\n    y_all = df_all[column_name].to_numpy()\n    X_all = df_all.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)#.to_numpy()\n    cat_features = [f for f in cat_vars if f in X_all.columns]\n    X_all2 = X_all.to_numpy()\n    X_solution = df_solution.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)\n    X_solution2 = X_solution.to_numpy()\n    # Select features\n    #sel = SelectFromModel(XGBClassifier(num_class=num_class, objective='multi:softmax', enable_categorical=False))\n    #sel.fit(X_all,y_all)\n    #selected_feat= X_all.columns[(sel.get_support())]\n    arr = np.zeros((len(num_class),len(num_class)))\n    score = 0\n    print(num_class)\n    y_solution = np.zeros(shape=(X_solution.shape[0],len(num_class)))\n    y_train = np.zeros(shape=(X_all.shape[0],len(num_class)))\n    skf = StratifiedKFold(n_splits=5,random_state=1990,shuffle=True)\n    for i, (train_index, test_index) in enumerate(skf.split(X_all, y_all)):\n        # Initialize model\n        models = [          \n           ('catBoost',make_pipeline(CatBoostClassifier(iterations=1000, verbose=1000, random_seed=42,learning_rate=0.05))),          \n           ('lgbm',make_pipeline(lgb.LGBMClassifier(random_state=42))),   \n           ('XGB',make_pipeline(XGBClassifier(objective='multi:softmax',random_state=42)))     \n            \n        ]\n        voter = VotingClassifier(models, voting = 'soft')\n        # Predict and evaluate\n        voter.fit(X_all.iloc[train_index], y_all[train_index])\n        y_pred = voter.predict_proba(X_all.iloc[test_index]) \n        \n        score += accuracy_score(y_all[test_index], np.argmax(y_pred, axis=1))/5\n        arr = arr + confusion_matrix(y_all[test_index], np.argmax(y_pred, axis=1))\n\n        # Predict solution\n        y_solution += voter.predict_proba(X_solution) #+ clf2.predict_proba(X_solution2)\n    \n    print('Model accuracy score: {0:0.4f}'. format(score))\n    #print(arr)\n    \n    return np.argmax(y_solution, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:13:32.656710Z","iopub.execute_input":"2024-02-27T00:13:32.657074Z","iopub.status.idle":"2024-02-27T00:13:32.672206Z","shell.execute_reply.started":"2024-02-27T00:13:32.657045Z","shell.execute_reply":"2024-02-27T00:13:32.670375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a copy of the solution dataframe for submission\ndf_submission = df_solution.copy()","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:13:37.288242Z","iopub.execute_input":"2024-02-27T00:13:37.288708Z","iopub.status.idle":"2024-02-27T00:13:37.298737Z","shell.execute_reply.started":"2024-02-27T00:13:37.288672Z","shell.execute_reply":"2024-02-27T00:13:37.297331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction q_noise_nearby","metadata":{}},{"cell_type":"code","source":"df_submission['q_noise_nearby'] = predict_any_column('q_noise_nearby', df_train, df_solution)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:13:41.640377Z","iopub.execute_input":"2024-02-27T00:13:41.641465Z","iopub.status.idle":"2024-02-27T00:15:20.160306Z","shell.execute_reply.started":"2024-02-27T00:13:41.641431Z","shell.execute_reply":"2024-02-27T00:15:20.159333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction q_noise_kind","metadata":{}},{"cell_type":"code","source":"df_submission['q_noise_kind'] = predict_any_column('q_noise_kind', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:15:20.161990Z","iopub.execute_input":"2024-02-27T00:15:20.162259Z","iopub.status.idle":"2024-02-27T00:18:45.796231Z","shell.execute_reply.started":"2024-02-27T00:15:20.162238Z","shell.execute_reply":"2024-02-27T00:18:45.795572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction q_earphones","metadata":{}},{"cell_type":"code","source":"df_submission['q_earphones'] = predict_any_column('q_earphones', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:18:45.797281Z","iopub.execute_input":"2024-02-27T00:18:45.797774Z","iopub.status.idle":"2024-02-27T00:20:54.555008Z","shell.execute_reply.started":"2024-02-27T00:18:45.797750Z","shell.execute_reply":"2024-02-27T00:20:54.554241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction q_thermal_preference","metadata":{}},{"cell_type":"code","source":"df_submission['q_thermal_preference'] = predict_any_column('q_thermal_preference', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:20:54.556766Z","iopub.execute_input":"2024-02-27T00:20:54.557748Z","iopub.status.idle":"2024-02-27T00:22:33.427793Z","shell.execute_reply.started":"2024-02-27T00:20:54.557722Z","shell.execute_reply":"2024-02-27T00:22:33.427135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Perform some postprocesiing\nThe labels were predicted independently, so, some combinations of the labels might not be possible, so this sample code try to fix few of these labels","metadata":{}},{"cell_type":"code","source":"# Convert classes back form integers to strings\nfor key in dict_cat:\n    if key in df_solution.columns and key in ['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']:\n        df_submission[key] = df_submission[key].replace(dict_cat[key])\n# Select only necessary columns for submission\nsub = df_submission[['id_unique', 'q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']].set_index('id_unique')","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:22:33.428648Z","iopub.execute_input":"2024-02-27T00:22:33.429595Z","iopub.status.idle":"2024-02-27T00:22:33.443788Z","shell.execute_reply.started":"2024-02-27T00:22:33.429571Z","shell.execute_reply":"2024-02-27T00:22:33.442505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Do a little bit of post processing\nfor index, row in sub.iterrows():\n    if row['q_noise_nearby'] == 'A little' and row['q_noise_kind'] == 'Not applicable' and row['q_earphones'] == 'Not applicable':\n        sub.at[index, 'q_earphones'] = 'No earphones'","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:22:33.445114Z","iopub.execute_input":"2024-02-27T00:22:33.445410Z","iopub.status.idle":"2024-02-27T00:22:33.676262Z","shell.execute_reply.started":"2024-02-27T00:22:33.445386Z","shell.execute_reply":"2024-02-27T00:22:33.675587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create submission file","metadata":{}},{"cell_type":"code","source":"# Write dataframe to csv file\nsub.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-27T00:22:33.677144Z","iopub.execute_input":"2024-02-27T00:22:33.677765Z","iopub.status.idle":"2024-02-27T00:22:33.691359Z","shell.execute_reply.started":"2024-02-27T00:22:33.677743Z","shell.execute_reply":"2024-02-27T00:22:33.690543Z"},"trusted":true},"execution_count":null,"outputs":[]}]}