{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from sklearn.model_selection import train_test_split, GridSearchCV,GroupKFold\nfrom sklearn.preprocessing import OrdinalEncoder\nimport warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np\nimport pandas as pd\nimport holidays\nimport time\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score,mean_absolute_percentage_error\nimport seaborn as sns\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:29:19.080815Z","iopub.execute_input":"2023-06-25T12:29:19.081327Z","iopub.status.idle":"2023-06-25T12:29:21.883003Z","shell.execute_reply.started":"2023-06-25T12:29:19.081289Z","shell.execute_reply":"2023-06-25T12:29:21.882031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import datasets\n\nssub= '/kaggle/input/dt-sumbission/sample_submission.csv'\ntrain= '/kaggle/input/dt-sumbission/train_data.csv'\ntest= '/kaggle/input/dt-sumbission/test_data.csv'\n\ntrain=pd.read_csv(train)\ntest=pd.read_csv(test)\nssub=pd.read_csv(ssub)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:29:21.888953Z","iopub.execute_input":"2023-06-25T12:29:21.889326Z","iopub.status.idle":"2023-06-25T12:29:53.813962Z","shell.execute_reply.started":"2023-06-25T12:29:21.889291Z","shell.execute_reply":"2023-06-25T12:29:53.813098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **EDA**","metadata":{}},{"cell_type":"code","source":"# Perform EDA with the use of pandas_profiling\n\n!pip install pandas-profiling > /dev/null\nfrom pandas_profiling import ProfileReport\nEDA=ProfileReport(train)\nEDA","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:29:53.823500Z","iopub.execute_input":"2023-06-25T12:29:53.824080Z","iopub.status.idle":"2023-06-25T12:38:11.268130Z","shell.execute_reply.started":"2023-06-25T12:29:53.824045Z","shell.execute_reply":"2023-06-25T12:38:11.267098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Takeaway points:\n\n* 6509155 vs 812478 for has_won==1\n* 23008 distinct in winBid\n* sentPrice = high correlation with winBid\n* Possible use of c1-c4 due the high correlation with other features\n* Most people use Android - sad truth\n* DT won ~12% of all auctions from the training set - not bad! ","metadata":{}},{"cell_type":"code","source":"# Check colinearity between continous variables\n\ncols=[]\nfor column in train.columns:\n    if train[column].dtypes == int or train[column].dtypes == np.int64 or train[column].dtypes ==np.float64:\n        cols.append(column)\n\nsns.heatmap(train[cols].corr(),annot=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:38:11.275187Z","iopub.execute_input":"2023-06-25T12:38:11.275650Z","iopub.status.idle":"2023-06-25T12:38:13.242644Z","shell.execute_reply.started":"2023-06-25T12:38:11.275621Z","shell.execute_reply":"2023-06-25T12:38:13.241481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"relationship between wind and sentPrice is not shocking. The rest of the features seem not to have a collinearity","metadata":{}},{"cell_type":"code","source":"# Check how many unique categories we have in each cetegorical variable\n\nfor i,v in train[['unitDisplayType','brandName','bundleId','appVersion','correctModelName','countryCode']].iteritems():\n    print(i,len(v.unique()))","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:38:13.255299Z","iopub.execute_input":"2023-06-25T12:38:13.255598Z","iopub.status.idle":"2023-06-25T12:38:16.434148Z","shell.execute_reply.started":"2023-06-25T12:38:13.255572Z","shell.execute_reply":"2023-06-25T12:38:16.432860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for NaN's'\npd.DataFrame(train.isna().sum(),columns=['Nans']).sort_values(by='Nans',ascending=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:38:16.494900Z","iopub.execute_input":"2023-06-25T12:38:16.495813Z","iopub.status.idle":"2023-06-25T12:38:29.903738Z","shell.execute_reply.started":"2023-06-25T12:38:16.495779Z","shell.execute_reply":"2023-06-25T12:38:29.902799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a Human Readable date and time out of timestamp\n\ndef create_HRD(df):\n    df['HRdate']=pd.to_datetime(df['eventTimestamp'], unit='ms').dt.date\n    df['HRtime']=pd.to_datetime(df['eventTimestamp'], unit='ms').dt.time\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:38:29.909040Z","iopub.execute_input":"2023-06-25T12:38:29.909465Z","iopub.status.idle":"2023-06-25T12:38:29.915337Z","shell.execute_reply.started":"2023-06-25T12:38:29.909427Z","shell.execute_reply":"2023-06-25T12:38:29.914451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# apply the function to create HRD and HRT\n\ntrain=create_HRD(train)\ntest=create_HRD(test)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:38:29.944759Z","iopub.execute_input":"2023-06-25T12:38:29.945965Z","iopub.status.idle":"2023-06-25T12:38:36.251881Z","shell.execute_reply.started":"2023-06-25T12:38:29.945922Z","shell.execute_reply":"2023-06-25T12:38:36.250658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the distribution if CountryCode == NaN and extract unique dates\n\ntrain[train['countryCode'].isna()==True]['HRdate'].value_counts()\nNans=train[train['countryCode'].isna()==True]['HRdate'].reset_index()\nNans=Nans['HRdate'].unique()\nNans","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:38:36.257193Z","iopub.execute_input":"2023-06-25T12:38:36.257528Z","iopub.status.idle":"2023-06-25T12:38:40.422120Z","shell.execute_reply.started":"2023-06-25T12:38:36.257499Z","shell.execute_reply":"2023-06-25T12:38:40.420758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The idea was to find a simple way to interpolate on NaNs in the countryCode column. The possibility that \"Unknown\" country is equal to the country with the highest prevalence of the requests on the same day, seems to be higher than simple forward fill.\n\n","metadata":{}},{"cell_type":"code","source":"# Find out which category was most frequent in case of the same dates\n\nfiltered_train = train[train['HRdate'].isin(Nans)]\ngrouped = filtered_train.groupby(['HRdate', 'countryCode'])\npd.DataFrame(grouped.size(),columns=['Freq']).groupby('HRdate').idxmax()\n\n# Fill NaNs with the country with highest propensity\n\ntrain['countryCode'].fillna('US',inplace=True)\n\n\n# important notice: relative distribution should be used during next iteration","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:38:43.996763Z","iopub.execute_input":"2023-06-25T12:38:43.997269Z","iopub.status.idle":"2023-06-25T12:38:46.112296Z","shell.execute_reply.started":"2023-06-25T12:38:43.997226Z","shell.execute_reply":"2023-06-25T12:38:46.111284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if countryCode NaNs were filled\n\npd.DataFrame(train.isna().sum(),columns=['Nans']).sort_values(by='Nans',ascending=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:38:46.117478Z","iopub.execute_input":"2023-06-25T12:38:46.117900Z","iopub.status.idle":"2023-06-25T12:39:01.517031Z","shell.execute_reply.started":"2023-06-25T12:38:46.117861Z","shell.execute_reply":"2023-06-25T12:39:01.515790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Feature Engineering**","metadata":{}},{"cell_type":"code","source":"def feature_engineering(group):\n\n\n# How many changes in app version were made compared to previous auction and how many changes were made overall\n    \n    group['switch_app'] = group['osAndVersion'].ne(group['osAndVersion'].shift()).astype(int)\n    group['switch_app_sum'] = group['osAndVersion'].nunique()\n\n# How many changes in phone model were made compared to previous auction and how many changes were made overall\n\n    group['switch_phone'] = group['correctModelName'].ne(group['correctModelName'].shift()).astype(int)\n    group['switch_phone_sum'] = group['correctModelName'].nunique()\n\n# How many changes in phone brand were made compared to previous auction and how many changes were made overall\n\n    group['switch_brand'] = group['brandName'].ne(group['brandName'].shift()).astype(int)\n    group['switch_brand_sum'] = group['brandName'].nunique()\n\n# Is current ad slot time is most common for this deviceId\n    \n    group['most_common_dt'] = (group['unitDisplayType'] == group['unitDisplayType'].value_counts().idxmax()).astype(int)\n\n# Is sent_price higher than mean\n    \n    group['sent_price_to_mean']=(group['sentPrice']>group['sentPrice'].mean()).astype(int)\n\n# Is sent_price higher than mean for the same day\n    \n    group['sent_price_to_mean_day'] = (group['sentPrice'] > group.groupby('HRdate')['sentPrice'].transform('mean')).astype(int)\n\n# Average sent price for this ID\n    \n    group['avg_sent_price']=group['sentPrice'].mean()\n\n# Average sent_price for this ID and day\n    \n    group['avg_sent_price_day']=group.groupby('HRdate')['sentPrice'].transform('mean')\n\n# Max sent_price for this ID\n    \n    group['max_sent_price']=group['sentPrice'].max()\n\n# Max sent_price for this ID and day\n    \n    group['max_sent_price_day']=group.groupby('HRdate')['sentPrice'].transform('max')\n\n# Date features\n    \n    group['HRdate']=pd.to_datetime(group['HRdate'])\n    group['Month'] = group['HRdate'].dt.month\n    group['Day'] = group['HRdate'].dt.day\n    group['Day_of_week'] = group['HRdate'].dt.dayofweek\n    group['Quarter'] = group['HRdate'].dt.quarter\n    group['Is_weekday'] = (group['HRdate'].dt.dayofweek < 5).astype(int)\n    group['is_weekend'] = group['HRdate'].dt.dayofweek.isin([5, 6]).astype(int)\n\n\n# Some features extracted from an anonymized columns\n    \n    group['dif_sent_floor'] = group['sentPrice'] - group['bidFloorPrice']\n    group['c2_plus_c4'] = group['c2'] + group['c4']\n    group['diff_c2_c4'] = group['c2'] - group['c4']\n    group['diff_c2_c4'] = group['c2'] / group['c4']\n    group['multop_c2_c4'] = group['c2'] * group['c4']\n\n    return group","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:39:01.530036Z","iopub.execute_input":"2023-06-25T12:39:01.530360Z","iopub.status.idle":"2023-06-25T12:39:01.543206Z","shell.execute_reply.started":"2023-06-25T12:39:01.530333Z","shell.execute_reply":"2023-06-25T12:39:01.541885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sort by date for each deviceID - necessery to extract correct features realted to the temporal order\n\ntrain=train.groupby('deviceId',as_index=False,sort=False).apply(lambda x: x.sort_values(by=['HRdate','HRtime'],ascending=True))\ntest=test.groupby('deviceId',as_index=False,sort=False).apply(lambda x: x.sort_values(by=['HRdate','HRtime'],ascending=True))\n\n# Apply Feature engineering function\n\ntrain,test=train.groupby('deviceId').apply(lambda x:feature_engineering(x)),test.groupby('deviceId').apply(lambda x:feature_engineering(x))","metadata":{"execution":{"iopub.status.busy":"2023-06-25T12:39:01.552192Z","iopub.execute_input":"2023-06-25T12:39:01.553218Z","iopub.status.idle":"2023-06-25T13:07:05.277251Z","shell.execute_reply.started":"2023-06-25T12:39:01.553185Z","shell.execute_reply":"2023-06-25T13:07:05.276256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reset_index\n\ntrain.reset_index(inplace=True)\ndel train['level_0']\ndel train['level_1']\ntest.reset_index(inplace=True)\ndel test['level_0']\ndel test['level_1']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save deviceId form test\n\nID_from_test=test['deviceId']","metadata":{"execution":{"iopub.status.busy":"2023-06-25T13:07:05.283852Z","iopub.execute_input":"2023-06-25T13:07:05.284219Z","iopub.status.idle":"2023-06-25T13:07:05.290027Z","shell.execute_reply.started":"2023-06-25T13:07:05.284183Z","shell.execute_reply":"2023-06-25T13:07:05.289022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the shape\n\nprint('Train shape', train.shape)\nprint('Test shape', test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T13:07:05.295439Z","iopub.execute_input":"2023-06-25T13:07:05.295879Z","iopub.status.idle":"2023-06-25T13:07:05.303904Z","shell.execute_reply.started":"2023-06-25T13:07:05.295845Z","shell.execute_reply":"2023-06-25T13:07:05.302754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Encoding**","metadata":{}},{"cell_type":"code","source":"# Encode categorical variables\n\nmerged_data = pd.concat([train, test], axis=0, ignore_index=True)\n\ncategorical_cols = []\nfor col_name, col_data in merged_data.iteritems():\n    if col_data.dtype == object and col_name != 'HRtime':\n        categorical_cols.append(col_name)\nmerged_data[categorical_cols] = merged_data[categorical_cols].astype(str)\n\n\nencoder = OrdinalEncoder()\nmerged_data[categorical_cols] = encoder.fit_transform(merged_data[categorical_cols])\n\n\ntrain = merged_data[:len(train)]\ntest = merged_data[len(train):]\n\ntest.drop(['has_won', 'winBid'], axis=1, inplace=True)\nprint(categorical_cols)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T13:07:05.311139Z","iopub.execute_input":"2023-06-25T13:07:05.311895Z","iopub.status.idle":"2023-06-25T13:08:35.927981Z","shell.execute_reply.started":"2023-06-25T13:07:05.311858Z","shell.execute_reply":"2023-06-25T13:08:35.926574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the shape again\n\nprint('Train shape', train.shape)\nprint('Test shape', test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T13:08:35.933286Z","iopub.execute_input":"2023-06-25T13:08:35.933582Z","iopub.status.idle":"2023-06-25T13:08:35.940366Z","shell.execute_reply.started":"2023-06-25T13:08:35.933550Z","shell.execute_reply":"2023-06-25T13:08:35.939257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model**","metadata":{}},{"cell_type":"markdown","source":"Selected estimator = Light Gradient Boosting Machine Regressor (typically way better than simple XGBoost on similar problems) with hyperparameter tuning based on GridSearchCV and groupKfold for balanced distribution of groups across the folds\n","metadata":{}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n\nestimator = lgb.LGBMRegressor(random_state=42,metric='l1')\n\n\nvalidation=[]\n\nparam_grid = {\n    'learning_rate': [0.01, 0.05, 0.1],\n    'n_estimators': [100, 150, 200],\n    'num_leaves': [10, 20, 30],\n    'max_depth': [5, 10],\n    'reg_alpha': [0, 0.1, 0.5],\n    'reg_lambda': [0, 0.1],\n    'min_child_samples': [10, 20, 30],\n    'subsample': [0.6, 0.7, 0.8]\n}\n\nn_splits = 3\n\ngroups = train['deviceId']\ngkf = GroupKFold(n_splits=n_splits)\n\nvalidation = []\n\nX_train = train.drop(labels=['has_won','winBid','HRdate','HRtime'], axis=1)\ny_train = train['winBid']\n\ncategorical_features_no = [X_train.columns.get_loc(col) for col in categorical_cols]\n\nfor train_index, test_index in gkf.split(X_train, y_train, groups):\n    X_train_split, X_val_split = X_train.iloc[train_index], X_train.iloc[test_index]\n    y_train_split, y_val_split = y_train.iloc[train_index], y_train.iloc[test_index]\n\n\n    # Perform GridSearchCV for Hyperparameters tuning\n    \n    grid_search = GridSearchCV(\n        estimator=estimator,\n        param_grid=param_grid,\n        cv=n_splits,\n        n_jobs=-1,\n        scoring='neg_root_mean_squared_error',\n        verbose=1,\n        return_train_score=True,\n    )\n\n    grid_search.fit(\n        X_train_split, y_train_split,\n        eval_set=[(X_val_split, y_val_split)],\n        eval_metric='‘neg_root_mean_squared_error',\n        early_stopping_rounds=50,\n        categorical_feature=categorical_features_no\n    )\n\n    # extract best hyperparameters found by minimizing of RMSE\n    cv_results = grid_search.cv_results_\n    mean_train_rmse = -np.mean(cv_results['mean_test_score'])\n    validation.append(mean_train_rmse)\n\n    if mean_train_rmse < best_rmse:\n        best_params = grid_search.best_params_\n        best_rmse = mean_train_rmse\n    \nfinal_model=lgb.LGBMRegressor(random_state=42,metric='l1',**best_params)\nfinal_model.fit(X_train, y_train)\n\n# Make predictions on the test data using the best model\ny_pred = final_model.predict(test.drop(['HRtime', 'HRdate'], axis=1))\n\n# Save the predictions \nfinal_predictions_df = pd.DataFrame({'predicted_winBid': y_pred})\nfinal_predictions_df.to_csv('Predictions.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T13:08:35.972586Z","iopub.execute_input":"2023-06-25T13:08:35.973474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test RMSE's from each fold\n\nvalidation","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Possible upgrade:\n\n# Remove sparse deviceID with less or equal than 5% of the rows for sentPrice variable (based on total distribution)\n\n# count_dID=pd.DataFrame(train.groupby('deviceId')['sentPrice'].count())\n# threshold=train.groupby('deviceId')['sentPrice'].count().quantile(0.05)\n# sparse_dID=count_dID[count_dID['sentPrice']<=threshold]\n\n# print(str(len(sparse_dID))+' devicesID will be removed, threshold = ' + str(int(threshold)) + ' rows')\n\n# sparse_idx=sparse_dID.index\n# train=train.query('deviceId not in @sparse_idx')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}