{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":61145,"databundleVersionId":7047669,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport pandas as pd \nimport numpy as np\nfrom datetime import datetime, timedelta\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.metrics import confusion_matrix\nfrom math import cos, asin, sqrt, pi\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\n\npd.set_option('display.max_columns', None)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-14T21:20:46.026315Z","iopub.execute_input":"2024-03-14T21:20:46.026754Z","iopub.status.idle":"2024-03-14T21:20:46.036021Z","shell.execute_reply.started":"2024-03-14T21:20:46.026712Z","shell.execute_reply":"2024-03-14T21:20:46.034781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**READ ALL FILES**\n\nRead all csv file","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_training.csv', index_col=[0], keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_solution = pd.read_csv('/kaggle/input/cool-quiet-city-competition/cozie_responses_and_physiological_data_test_public.csv', index_col=[0], keep_default_na=False, na_values=[''], parse_dates=['time', 'ws_timestamp_location', 'ws_timestamp_start'], date_format='%Y-%m-%d %H:%M:%S.%f%z')\ndf_stations = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_stations.csv').drop(columns=['Unnamed: 0'])\ndf_rain = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_rainfall.csv', parse_dates=True, index_col=[0])\ndf_temperature = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_air-temperature.csv', parse_dates=True, index_col=[0])\ndf_humidity = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_relative-humidity.csv', parse_dates=True, index_col=[0])\ndf_wind_speed = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-speed.csv', parse_dates=True, index_col=[0])\ndf_wind_direction = pd.read_csv('/kaggle/input/cool-quiet-city-competition/weather_wind-direction.csv', parse_dates=True, index_col=[0])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:20:49.424994Z","iopub.execute_input":"2024-03-14T21:20:49.425428Z","iopub.status.idle":"2024-03-14T21:23:09.039128Z","shell.execute_reply.started":"2024-03-14T21:20:49.425396Z","shell.execute_reply":"2024-03-14T21:23:09.037843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Two new DataFrames**\n\nsine and cosine values are assigned to the corresponding columns in the DataFrames","metadata":{}},{"cell_type":"code","source":"df_wind_direction_sin = df_wind_direction.copy(deep=True)\ndf_wind_direction_cos = df_wind_direction.copy(deep=True)\nfor s in df_wind_direction.columns:\n    df_wind_direction_sin[s] = np.sin(np.pi/180*df_wind_direction_sin[s])\n    df_wind_direction_cos[s] = np.cos(np.pi/180*df_wind_direction_cos[s])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:23:28.262711Z","iopub.execute_input":"2024-03-14T21:23:28.263108Z","iopub.status.idle":"2024-03-14T21:23:28.523677Z","shell.execute_reply.started":"2024-03-14T21:23:28.263078Z","shell.execute_reply":"2024-03-14T21:23:28.521993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training and testing data are used to finally merge them into one.\nIn my experience as an Industrial Engineering student, a percentage of 70% of the data is used for training and 30% of the data for testing.","metadata":{}},{"cell_type":"code","source":"df_train['source']='train'\ndf_solution['source']='solution'\ndf_all = pd.concat([df_train, df_solution])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:23:30.660813Z","iopub.execute_input":"2024-03-14T21:23:30.661193Z","iopub.status.idle":"2024-03-14T21:23:31.157942Z","shell.execute_reply.started":"2024-03-14T21:23:30.661164Z","shell.execute_reply":"2024-03-14T21:23:31.156314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Several new columns are being created in the DataFrame `df_all` based on the values of existing columns.\nThe purpose of creating these columns is to extract categorical information from the original columns and possibly use them as features for analysis or modeling tasks. This process helps in organizing and structuring the data for further analysis.","metadata":{}},{"cell_type":"code","source":"df_all['q_Indoor'] = df_all['q_location'].str.contains(\"Indoor\")\ndf_all['q_Class'] = df_all['q_location'].isin(['Indoor - Class'])\ndf_all['q_Home'] = df_all['q_location'].isin(['Indoor - Home'])\ndf_all['q_Office'] = df_all['q_location'].isin(['Indoor - Office'])\ndf_all['q_Indoor_Other'] = df_all['q_location'].isin(['Indoor - Other'])\ndf_all['q_Outdoor'] = df_all['q_location'].isin(['Outdoor'])\ndf_all['q_Transportation'] = df_all['q_location'].isin(['Transportation'])\ndf_all['q_alone_group'] = df_all['q_alone_group'].isin(['Alone'])\ndf_all['q_Bus'] = df_all['q_location_transport'].isin(['Bus'])\ndf_all['q_Car'] = df_all['q_location_transport'].isin(['Car'])\ndf_all['q_Taxi'] = df_all['q_location_transport'].isin(['Taxi'])\ndf_all['q_Train'] = df_all['q_location_transport'].isin(['Train'])\ndf_all['q_Transport_Other'] = df_all['q_location_transport'].isin(['Other'])\ndf_all['q_Focus'] = df_all['q_activity_category_alone'].isin(['Focus'])\ndf_all['q_Leisure'] = df_all['q_activity_category_alone'].isin(['Leisure'])\ndf_all['q_Alone_Activity_Other'] = df_all['q_activity_category_alone'].isin(['Other'])\ndf_all['q_Collaborate'] = df_all['q_activity_category_group'].isin(['Collaborate'])\ndf_all['q_Learn'] = df_all['q_activity_category_group'].isin(['Learn'])\ndf_all['q_Socialize'] = df_all['q_activity_category_group'].isin(['Socialize'])\ndf_all['q_Group_Activity_Other'] = df_all['q_activity_category_group'].isin(['Other'])\ndf_all['q_Conference room'] = df_all['q_location_office'].isin(['Conference room'])\ndf_all['q_Cubicles'] = df_all['q_location_office'].isin(['Cubicles'])\ndf_all['q_Individual'] = df_all['q_location_office'].isin(['Individual'])\ndf_all['q_Large open plan'] = df_all['q_location_office'].isin(['Large open plan'])\ndf_all['q_Not applicable'] = df_all['q_location_office'].isin(['Not applicable'])\ndf_all['q_Small shared'] = df_all['q_location_office'].isin(['Small shared'])\n\ndf_all = df_all.drop(columns=['q_location','q_location_transport','q_activity_category_alone','q_activity_category_group','q_location_office'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:23:34.885661Z","iopub.execute_input":"2024-03-14T21:23:34.886082Z","iopub.status.idle":"2024-03-14T21:23:37.726618Z","shell.execute_reply.started":"2024-03-14T21:23:34.886049Z","shell.execute_reply":"2024-03-14T21:23:37.725487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"the \"duration\" column provides a measure of the duration of each event on a logarithmic scale of seconds, which can be useful for analyzing and comparing the duration of different events in the dataset.","metadata":{}},{"cell_type":"code","source":"df_all['duration'] = np.log((df_all.index - df_all['ws_timestamp_start']).dt.total_seconds())","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:23:48.128814Z","iopub.execute_input":"2024-03-14T21:23:48.129358Z","iopub.status.idle":"2024-03-14T21:23:48.214741Z","shell.execute_reply.started":"2024-03-14T21:23:48.129315Z","shell.execute_reply":"2024-03-14T21:23:48.213110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute day and time features\ndf_all['time_of_day_cos'] = np.cos(np.deg2rad((df_all.index.hour*60 + df_all.index.minute)/4))\ndf_all['time_of_day_sin'] = np.sin(np.deg2rad((df_all.index.hour*60 + df_all.index.minute)/4))\ndf_all['time_of_day'] = (df_all.index.hour*60 + df_all.index.minute)\ndf_all['day_of_week'] = df_all.index.dayofweek\n#df_all = df_all.reset_index()\n#df_all = df_all.sort_values(['id_participant', 'time'])\n#df_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:23:50.213465Z","iopub.execute_input":"2024-03-14T21:23:50.213970Z","iopub.status.idle":"2024-03-14T21:23:50.958718Z","shell.execute_reply.started":"2024-03-14T21:23:50.213930Z","shell.execute_reply":"2024-03-14T21:23:50.957525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This block of code is calculating several summary statistics for different health-related and physical behavior variables. \n\nThese summary measures can provide a general sense of trends and variabilities in the data related to participants' health and physical behavior.","metadata":{}},{"cell_type":"code","source":"df_all['med_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('median')\ndf_all['med_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('median')\ndf_all['med_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('median')\ndf_all['med_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('median')\ndf_all['med_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('median')\ndf_all['med_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('median')\ndf_all['med_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('median')\n\ndf_all['mean_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('mean')\ndf_all['mean_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('mean')\ndf_all['mean_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('mean')\ndf_all['mean_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('mean')\ndf_all['mean_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('mean')\ndf_all['mean_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('mean')\ndf_all['mean_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('mean')\n\ndf_all['std_resting_heart_rate'] = df_all.groupby('id_participant')['ts_resting_heart_rate'].transform('std')\ndf_all['std_audio_exposure_environment'] = df_all.groupby('id_participant')['ts_audio_exposure_environment'].transform('std')\ndf_all['std_oxygen_saturation'] = df_all.groupby('id_participant')['ts_oxygen_saturation'].transform('std')\ndf_all['std_stand_time'] = df_all.groupby('id_participant')['ts_stand_time'].transform('std')\ndf_all['std_heart_rate'] = df_all.groupby('id_participant')['ts_heart_rate'].transform('std')\ndf_all['std_step_count'] = df_all.groupby('id_participant')['ts_step_count'].transform('std')\ndf_all['std_walking_distance'] = df_all.groupby('id_participant')['ts_walking_distance'].transform('std')","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:23:53.054299Z","iopub.execute_input":"2024-03-14T21:23:53.056024Z","iopub.status.idle":"2024-03-14T21:23:58.276192Z","shell.execute_reply.started":"2024-03-14T21:23:53.055958Z","shell.execute_reply":"2024-03-14T21:23:58.274798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['ts_resting_heart_rate'] = (df_all['ts_resting_heart_rate'] - df_all['mean_resting_heart_rate'])/df_all['std_resting_heart_rate']\ndf_all['ts_audio_exposure_environment'] = (df_all['ts_audio_exposure_environment'] - df_all['mean_audio_exposure_environment'])/df_all['std_audio_exposure_environment']\ndf_all['ts_oxygen_saturation'] = (df_all['ts_oxygen_saturation'] - df_all['mean_oxygen_saturation'])/df_all['std_oxygen_saturation']\ndf_all['ts_stand_time'] = (df_all['ts_stand_time'] - df_all['mean_stand_time'])/df_all['std_stand_time']\ndf_all['ts_heart_rate'] = (df_all['ts_heart_rate'] - df_all['mean_heart_rate'])/df_all['std_heart_rate']\ndf_all['ts_step_count'] = (df_all['ts_step_count'] - df_all['mean_step_count'])/df_all['std_step_count']\ndf_all['ts_walking_distance'] = (df_all['ts_walking_distance'] - df_all['mean_walking_distance'])/df_all['std_walking_distance']","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:24:02.133367Z","iopub.execute_input":"2024-03-14T21:24:02.133805Z","iopub.status.idle":"2024-03-14T21:24:02.298656Z","shell.execute_reply.started":"2024-03-14T21:24:02.133771Z","shell.execute_reply":"2024-03-14T21:24:02.296683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**INTERPOLATION**\n\nThis function is designed to resample health data to align with the timestamps of watch survey responses.","metadata":{}},{"cell_type":"code","source":"# Function to resample health data at the timestamps of watch survey responses.\ndef resample_health(df, col_name, col_name_new):\n    df[col_name_new] = df[col_name].interpolate(method='linear').bfill().ffill().mask(df['ws_survey_count'].isna())\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:24:05.093479Z","iopub.execute_input":"2024-03-14T21:24:05.094024Z","iopub.status.idle":"2024-03-14T21:24:05.101334Z","shell.execute_reply.started":"2024-03-14T21:24:05.093969Z","shell.execute_reply":"2024-03-14T21:24:05.100181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resample normalized health data\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_heart_rate', col_name_new='ws_heart_rate').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_audio_exposure_environment', col_name_new='ws_audio_exposure_environment').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_oxygen_saturation', col_name_new='ws_oxygen_saturation').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_resting_heart_rate', col_name_new='ws_resting_heart_rate').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_stand_time', col_name_new='ws_stand_time').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_step_count', col_name_new='ws_step_count').reset_index(level=[0], drop=True)\ndf_all = df_all.groupby('id_participant').apply(resample_health, col_name='ts_walking_distance', col_name_new='ws_walking_distance').reset_index(level=[0], drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:24:07.133472Z","iopub.execute_input":"2024-03-14T21:24:07.133963Z","iopub.status.idle":"2024-03-14T21:24:49.052115Z","shell.execute_reply.started":"2024-03-14T21:24:07.133927Z","shell.execute_reply":"2024-03-14T21:24:49.050666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**About the message above:**\n\nThese deprecation warnings indicate that you're using the apply method on a DataFrameGroupBy object that operates on the grouping columns. This behavior is deprecated, and in future versions of pandas, the grouping columns will be excluded from the operation.","metadata":{}},{"cell_type":"markdown","source":"## Interpolate weather data\n\nThese functions are designed to retrieve weather data from the nearest weather station for a specific timestamp and location while considering a specified window of time and distance.","metadata":{}},{"cell_type":"code","source":"# Function to get weather data for a given timestamp and location from the nearest weather station\ndef get_weather_data(df_weather_data, lat, lon, timestamp):    \n    # Get timestamp with no seconds or microseconds\n    timestamp = timestamp.replace(second=0, microsecond=0)\n    \n    # Get weather data for timestamp above\n    weather_data_row = df_weather_data[(df_weather_data.index >= timestamp- timedelta(hours=0, minutes=10)) & (df_weather_data.index <= timestamp + timedelta(hours=0, minutes=10))]\n\n    # If no weather data is available for timestamp above, get weather data for next minute\n    if (weather_data_row.empty):\n        timestamp = timestamp + pd.Timedelta(minutes=1)\n        weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n        # If there is still no weather data available, return NaN\n        if (weather_data_row.empty):\n            timestamp = timestamp - 2 * pd.Timedelta(minutes=1)\n            weather_data_row = df_weather_data[df_weather_data.index == timestamp]\n            if (weather_data_row.empty):\n                return np.NaN\n\n    # Get weather station ids from weather stations with with data\n    station_ids_with_data = weather_data_row.columns[weather_data_row.count() > 0].tolist()\n\n    # Get weather stations with data\n    df_stations2 = df_stations[df_stations.id.isin(station_ids_with_data)]\n\n    # Compute distance to each weather station\n    df_stations2['distance'] = df_stations2.apply(lambda x: distance(x['latitude'], x['longitude'], lat, lon), axis=1)\n\n    # Get nearest weather station\n    station_id_nearest = df_stations2[df_stations2.distance==df_stations2.distance.min()].id.values[0]\n\n    # Retrun NaN for cases where the nearest weather station is more than 10km away\n    if df_stations2.distance.min()>10:\n        return np.NaN\n\n    # Return weather data from nearest weather station\n    return np.nanmean(weather_data_row[station_id_nearest])\n\n# Function to compute distnace on spherical earth\ndef distance(lat1, lon1, lat2, lon2):\n    r = 6371 # km\n    p = pi / 180\n\n    a = 0.5 - cos((lat2-lat1)*p)/2 + cos(lat1*p) * cos(lat2*p) * (1-cos((lon2-lon1)*p))/2\n    return 2 * r * asin(sqrt(a))","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:26:05.066578Z","iopub.execute_input":"2024-03-14T21:26:05.067044Z","iopub.status.idle":"2024-03-14T21:26:05.083145Z","shell.execute_reply.started":"2024-03-14T21:26:05.067012Z","shell.execute_reply":"2024-03-14T21:26:05.081628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply above function to the dataset for all weather data\ndf_all = df_all.reset_index()\ndf_all['ws_rainfall'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_rain, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_air_temperature'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_temperature, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_relative_humidity'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_humidity, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_speed'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_speed, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_sin'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_sin, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all['ws_wind_direction_cos'] = df_all[df_all.ws_survey_count.notna()].apply(lambda row: get_weather_data(df_wind_direction_cos, row['ws_latitude'], row['ws_longitude'], row['time']), axis=1)\ndf_all = df_all.set_index('time', drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:26:08.150584Z","iopub.execute_input":"2024-03-14T21:26:08.151040Z","iopub.status.idle":"2024-03-14T21:32:04.404873Z","shell.execute_reply.started":"2024-03-14T21:26:08.151007Z","shell.execute_reply":"2024-03-14T21:32:04.403707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Remove rowns and columns**","metadata":{}},{"cell_type":"code","source":"# Remove all rows that are not a watch surve response\ndf_all = df_all[df_all.ws_survey_count.notna()]\n\n# Remove some timestamp columns\ndf_all = df_all.drop(columns=['ws_timestamp_location', 'ws_timestamp_start'])\n\n# Remove columns of the original health data those are no longer needed since we are going to use the interpolated values\ndf_all = df_all.drop(columns=['ts_heart_rate',\n                              'ts_audio_exposure_environment',\n                              'ts_oxygen_saturation',\n                              'ts_resting_heart_rate',\n                              'ts_stand_time', \n                              'ts_step_count',\n                              'ts_walking_distance'])\n\n# These operations help clean and prepare the DataFrame for further analysis by removing rows and columns that are not relevant for analysis.","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:32:21.897389Z","iopub.execute_input":"2024-03-14T21:32:21.897905Z","iopub.status.idle":"2024-03-14T21:32:22.024734Z","shell.execute_reply.started":"2024-03-14T21:32:21.897866Z","shell.execute_reply":"2024-03-14T21:32:22.022814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"earphone_dict = df_all.groupby(['id_participant','q_earphones'])['duration'].count().to_dict()\ndf_all['own_earphones'] = 0\ndf_all['own_noise_cancelling'] = 0\nfor index, row in df_all.iterrows():\n    if (row['id_participant'], 'Earphones') in earphone_dict:\n        df_all.at[index, 'own_earphones'] = 1\n    if (row['id_participant'], 'Noise cancelling') in earphone_dict:\n        df_all.at[index, 'own_noise_cancelling'] = 1","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:32:51.725949Z","iopub.execute_input":"2024-03-14T21:32:51.726507Z","iopub.status.idle":"2024-03-14T21:32:54.211903Z","shell.execute_reply.started":"2024-03-14T21:32:51.726469Z","shell.execute_reply":"2024-03-14T21:32:54.210326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Use_earphone_in_class = set(df_all[(df_all['q_Class']) & (df_all['q_earphones']=='Earphones')]['id_participant'].unique())\nbeen_to_class = set(df_all[(df_all['q_Class'])&(df_all['source'] == 'train')]['id_participant'].unique())\ndiff = been_to_class.difference(Use_earphone_in_class)\nfor index, row in df_all.iterrows():\n    if row['q_Class']:\n        df_all.at[index, 'own_noise_cancelling'] = 0\n        if row['id_participant'] in diff:\n            df_all.at[index, 'own_earphones'] = 0","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:33:01.932145Z","iopub.execute_input":"2024-03-14T21:33:01.932768Z","iopub.status.idle":"2024-03-14T21:33:02.874860Z","shell.execute_reply.started":"2024-03-14T21:33:01.932723Z","shell.execute_reply":"2024-03-14T21:33:02.873394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['location_count'] = 0\nfor index, row in df_all.iterrows():\n    df_all.at[index, 'location_count'] = len(df_all[(df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:33:04.770572Z","iopub.execute_input":"2024-03-14T21:33:04.771185Z","iopub.status.idle":"2024-03-14T21:33:18.796605Z","shell.execute_reply.started":"2024-03-14T21:33:04.771138Z","shell.execute_reply":"2024-03-14T21:33:18.795356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['location_count'] = 0\nfor index, row in df_all.iterrows():\n    df_all.at[index, 'location_count'] = len(df_all[(df_all['ws_latitude'] == row['ws_latitude']) & (df_all['ws_longitude'] == row['ws_longitude'])])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:33:26.391248Z","iopub.execute_input":"2024-03-14T21:33:26.391825Z","iopub.status.idle":"2024-03-14T21:33:40.561645Z","shell.execute_reply.started":"2024-03-14T21:33:26.391780Z","shell.execute_reply":"2024-03-14T21:33:40.560075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separte training and solution data again\ndf_train = df_all[df_all.source == 'train'].copy()\ndf_solution = df_all[df_all.source == 'solution'].copy()\n\n# Drop the helper columns ws_surve_count_2 and source again\ndf_train = df_train.drop(columns=['source'])\ndf_solution = df_solution.drop(columns=['source'])","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:33:47.321115Z","iopub.execute_input":"2024-03-14T21:33:47.321607Z","iopub.status.idle":"2024-03-14T21:33:47.449237Z","shell.execute_reply.started":"2024-03-14T21:33:47.321572Z","shell.execute_reply":"2024-03-14T21:33:47.447656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**COOKING THE DATA**\n\nThis code block aims to preprocess categorical variables in the training and solution DataFrames (df_train and df_solution, respectively) by encoding them as numerical values.","metadata":{}},{"cell_type":"code","source":"# Find columns with values that are not numeric\ncat_vars = list(df_train.select_dtypes(include=['object']).columns)\n# encoding categorical variables\ndict_cat = {}\n\nfor var in cat_vars:\n    if var == 'id_participant': # no need to encode the participant ID\n        continue\n    # convert categorical variables into 'category' type\n    df_train[var] = df_train[var].astype('category')\n    df_solution[var] = df_solution[var].astype('category')\n    \n    # dictionary to keep track of the variable mapping: {variable-name: {number1: string1, number2: string2, ...}\n    dict_cat[var] = dict(enumerate(df_train[var].cat.categories))\n\n    # encode variables, missing values are encoded as -1\n    df_train[var] = df_train[var].cat.codes\n    df_solution[var] = df_solution[var].cat.codes\n    \n    # After executing this code block, the categorical variables in both the training and solution DataFrames will be encoded as numerical values, making them suitable for machine learning algorithms that require numerical input.","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:33:49.794162Z","iopub.execute_input":"2024-03-14T21:33:49.795607Z","iopub.status.idle":"2024-03-14T21:33:49.825868Z","shell.execute_reply.started":"2024-03-14T21:33:49.795539Z","shell.execute_reply":"2024-03-14T21:33:49.824897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This code imports necessary classes from the scikit-learn library for building machine learning pipelines and ensemble classifiers.\n\nWith these classes imported, you can create pipelines, ensemble classifiers, and stacking classifiers for your machine learning tasks.","metadata":{}},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, BaggingClassifier\nfrom sklearn.ensemble import VotingClassifier, StackingClassifier","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:33:52.073084Z","iopub.execute_input":"2024-03-14T21:33:52.074010Z","iopub.status.idle":"2024-03-14T21:33:52.188610Z","shell.execute_reply.started":"2024-03-14T21:33:52.073972Z","shell.execute_reply":"2024-03-14T21:33:52.187380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_any_column(column_name, df_all, df_solution):\n    # Remove rows where the predicting variable is missing\n    df_all = df_all[df_all[column_name] != -1]\n    \n    # Remove rows where the predicting variable is missing\n    df_all = df_all.drop(columns=['id_participant'])\n    df_solution = df_solution.drop(columns=['id_participant'])\n    \n    # Count classes in predicting variable\n    num_class = df_train[column_name].unique()\n    \n    # Break dataframe into variables and ground truth\n    y_all = df_all[column_name].to_numpy()\n    X_all = df_all.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)#.to_numpy()\n    cat_features = [f for f in cat_vars if f in X_all.columns]\n    X_all2 = X_all.to_numpy()\n    X_solution = df_solution.drop(['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference'], axis=1)\n    X_solution2 = X_solution.to_numpy()\n    # Select features\n    #sel = SelectFromModel(XGBClassifier(num_class=num_class, objective='multi:softmax', enable_categorical=False))\n    #sel.fit(X_all,y_all)\n    #selected_feat= X_all.columns[(sel.get_support())]\n    arr = np.zeros((len(num_class),len(num_class)))\n    score = 0\n    print(num_class)\n    y_solution = np.zeros(shape=(X_solution.shape[0],len(num_class)))\n    y_train = np.zeros(shape=(X_all.shape[0],len(num_class)))\n    skf = StratifiedKFold(n_splits=5,random_state=1990,shuffle=True)\n    for i, (train_index, test_index) in enumerate(skf.split(X_all, y_all)):\n        # Initialize model\n        models = [          \n           ('catBoost',make_pipeline(CatBoostClassifier(iterations=1000, verbose=1000, random_seed=42,learning_rate=0.05))),          \n           ('lgbm',make_pipeline(lgb.LGBMClassifier(random_state=42))),   \n           ('XGB',make_pipeline(XGBClassifier(objective='multi:softmax',random_state=42)))     \n            \n        ]\n        voter = VotingClassifier(models, voting = 'soft')\n        # Predict and evaluate\n        voter.fit(X_all.iloc[train_index], y_all[train_index])\n        y_pred = voter.predict_proba(X_all.iloc[test_index]) \n        \n        score += accuracy_score(y_all[test_index], np.argmax(y_pred, axis=1))/5\n        arr = arr + confusion_matrix(y_all[test_index], np.argmax(y_pred, axis=1))\n\n        # Predict solution\n        y_solution += voter.predict_proba(X_solution) #+ clf2.predict_proba(X_solution2)\n    \n    print('Model accuracy score: {0:0.4f}'. format(score))\n    #print(arr)\n    \n    return np.argmax(y_solution, axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:33:53.745796Z","iopub.execute_input":"2024-03-14T21:33:53.746209Z","iopub.status.idle":"2024-03-14T21:33:53.764305Z","shell.execute_reply.started":"2024-03-14T21:33:53.746177Z","shell.execute_reply":"2024-03-14T21:33:53.762651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a copy of the solution dataframe for submission\ndf_submission = df_solution.copy()","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:33:58.082354Z","iopub.execute_input":"2024-03-14T21:33:58.082815Z","iopub.status.idle":"2024-03-14T21:33:58.091132Z","shell.execute_reply.started":"2024-03-14T21:33:58.082781Z","shell.execute_reply":"2024-03-14T21:33:58.089434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_noise_nearby'] = predict_any_column('q_noise_nearby', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:34:01.464612Z","iopub.execute_input":"2024-03-14T21:34:01.465027Z","iopub.status.idle":"2024-03-14T21:36:45.472933Z","shell.execute_reply.started":"2024-03-14T21:34:01.464995Z","shell.execute_reply":"2024-03-14T21:36:45.471417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_noise_kind'] = predict_any_column('q_noise_kind', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:36:50.570627Z","iopub.execute_input":"2024-03-14T21:36:50.571127Z","iopub.status.idle":"2024-03-14T21:42:24.717646Z","shell.execute_reply.started":"2024-03-14T21:36:50.571092Z","shell.execute_reply":"2024-03-14T21:42:24.716047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_earphones'] = predict_any_column('q_earphones', df_train, df_solution)","metadata":{"execution":{"iopub.status.busy":"2024-03-14T21:42:28.235941Z","iopub.execute_input":"2024-03-14T21:42:28.236474Z","iopub.status.idle":"2024-03-14T21:45:05.392371Z","shell.execute_reply.started":"2024-03-14T21:42:28.236437Z","shell.execute_reply":"2024-03-14T21:45:05.390172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['q_thermal_preference'] = predict_any_column('q_thermal_preference', df_train, df_solution)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert classes back form integers to strings\nfor key in dict_cat:\n    if key in df_solution.columns and key in ['q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']:\n        df_submission[key] = df_submission[key].replace(dict_cat[key])\n# Select only necessary columns for submission\nsub = df_submission[['id_unique', 'q_noise_nearby', 'q_noise_kind', 'q_earphones', 'q_thermal_preference']].set_index('id_unique')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Do a little bit of post processing\nfor index, row in sub.iterrows():\n    if row['q_noise_nearby'] == 'A little' and row['q_noise_kind'] == 'Not applicable' and row['q_earphones'] == 'Not applicable':\n        sub.at[index, 'q_earphones'] = 'No earphones'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Write dataframe to csv file\nsub.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}