{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.base import TransformerMixin\nfrom sklearn.metrics import mean_squared_error\nfrom lightgbm import LGBMRegressor\nfrom dataclasses import dataclass\nfrom category_encoders import OrdinalEncoder\nfrom sklearn.model_selection import GroupKFold\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport optuna\nimport joblib\nimport time\nimport warnings\n\n\nfrom copy import deepcopy\n\npd.options.mode.chained_assignment = None\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.simplefilter(action='ignore', category=DeprecationWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-10T06:39:03.920491Z","iopub.execute_input":"2023-06-10T06:39:03.920986Z","iopub.status.idle":"2023-06-10T06:39:07.649574Z","shell.execute_reply.started":"2023-06-10T06:39:03.920945Z","shell.execute_reply":"2023-06-10T06:39:07.648265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"@dataclass\nclass CONFIG:\n    data_path = '../input/digital-turbine-auction-bid-price-prediction/'\n    sample = None\n    params = {'num_leaves': 254,\n             'lambda_l1': 0.0019139179939626659,\n             'lambda_l2': 1.2097079649236921e-05,\n             'feature_fraction': 0.9094400437591823,\n             'bagging_fraction': 0.8282871046695927,\n             'bagging_freq': 3,\n             'min_child_samples': 51,\n             'boosting_type': 'dart',\n             'objective': 'rmse',\n             'metric': 'rmse',\n             'num_iterations': 500,\n             'verbose': -1}","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:39:07.651940Z","iopub.execute_input":"2023-06-10T06:39:07.652548Z","iopub.status.idle":"2023-06-10T06:39:07.661278Z","shell.execute_reply.started":"2023-06-10T06:39:07.652502Z","shell.execute_reply":"2023-06-10T06:39:07.659936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(config):\n    train_df = pd.read_csv(f'{config.data_path}train_data.csv', nrows=config.sample)\n    test_df = pd.read_csv(f'{config.data_path}test_data.csv')\n    sub = pd.read_csv(f'{config.data_path}sample_submission.csv')\n    return train_df, test_df, sub","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:39:07.662461Z","iopub.execute_input":"2023-06-10T06:39:07.662792Z","iopub.status.idle":"2023-06-10T06:39:07.678164Z","shell.execute_reply.started":"2023-06-10T06:39:07.662765Z","shell.execute_reply":"2023-06-10T06:39:07.676736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## First look","metadata":{}},{"cell_type":"code","source":"train_df, test_df, sub = load_data(CONFIG)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:39:10.579525Z","iopub.execute_input":"2023-06-10T06:39:10.579999Z","iopub.status.idle":"2023-06-10T06:39:59.011052Z","shell.execute_reply.started":"2023-06-10T06:39:10.579965Z","shell.execute_reply":"2023-06-10T06:39:59.009837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:16.756041Z","iopub.execute_input":"2023-06-09T17:35:16.756360Z","iopub.status.idle":"2023-06-09T17:35:16.786154Z","shell.execute_reply.started":"2023-06-09T17:35:16.756333Z","shell.execute_reply":"2023-06-09T17:35:16.785022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:16.787509Z","iopub.execute_input":"2023-06-09T17:35:16.787851Z","iopub.status.idle":"2023-06-09T17:35:16.814628Z","shell.execute_reply.started":"2023-06-09T17:35:16.787822Z","shell.execute_reply":"2023-06-09T17:35:16.813455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape, test_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:16.816759Z","iopub.execute_input":"2023-06-09T17:35:16.817436Z","iopub.status.idle":"2023-06-09T17:35:16.824895Z","shell.execute_reply.started":"2023-06-09T17:35:16.817402Z","shell.execute_reply":"2023-06-09T17:35:16.823718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Missing values","metadata":{}},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:16.826582Z","iopub.execute_input":"2023-06-09T17:35:16.827045Z","iopub.status.idle":"2023-06-09T17:35:16.882696Z","shell.execute_reply.started":"2023-06-09T17:35:16.827006Z","shell.execute_reply":"2023-06-09T17:35:16.881232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:16.886254Z","iopub.execute_input":"2023-06-09T17:35:16.886604Z","iopub.status.idle":"2023-06-09T17:35:17.066647Z","shell.execute_reply.started":"2023-06-09T17:35:16.886577Z","shell.execute_reply":"2023-06-09T17:35:17.065535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Time\n\nThe dataset is a set of `deviceId`-level time series. The test set consist of the last event for every `deviceId`. Events for the test set could happen earlier than some for the train set, but only for different `deviceId`. There are `1410` `deviceId` in the test set that are not present in the train set.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2, figsize=(15,10))\nsns.kdeplot(train_df.eventTimestamp, ax=ax[0])\nsns.kdeplot(test_df.eventTimestamp, ax=ax[0])\n\nsns.histplot(train_df.eventTimestamp, ax=ax[1])\nsns.histplot(test_df.eventTimestamp, ax=ax[1]);","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:17.069911Z","iopub.execute_input":"2023-06-09T17:35:17.070336Z","iopub.status.idle":"2023-06-09T17:35:18.561544Z","shell.execute_reply.started":"2023-06-09T17:35:17.070305Z","shell.execute_reply":"2023-06-09T17:35:18.560286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.deviceId.is_unique","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:18.562948Z","iopub.execute_input":"2023-06-09T17:35:18.563310Z","iopub.status.idle":"2023-06-09T17:35:18.583216Z","shell.execute_reply.started":"2023-06-09T17:35:18.563279Z","shell.execute_reply":"2023-06-09T17:35:18.581382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.deviceId = train_df.deviceId.astype(str)\ntrain_df.eventTimestamp = train_df.eventTimestamp.astype(np.int64)\nmax_eventTimestamp_train = train_df.loc[:, ['deviceId', 'eventTimestamp']].groupby('deviceId').max()['eventTimestamp']\ntest_df_timecheck = test_df.loc[:, ['deviceId', 'eventTimestamp']]\ntest_df_timecheck['eventTimestamp_train_max'] = test_df_timecheck.deviceId.map(max_eventTimestamp_train)\ntest_df_timecheck['before'] = (test_df_timecheck.eventTimestamp_train_max < test_df_timecheck.eventTimestamp)\ntest_df_timecheck.loc[~test_df_timecheck.eventTimestamp_train_max.isna()].before.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:18.584737Z","iopub.execute_input":"2023-06-09T17:35:18.585263Z","iopub.status.idle":"2023-06-09T17:35:18.627325Z","shell.execute_reply.started":"2023-06-09T17:35:18.585230Z","shell.execute_reply":"2023-06-09T17:35:18.626242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df_timecheck.loc[test_df_timecheck.eventTimestamp_train_max.isna(), 'deviceId'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:18.629140Z","iopub.execute_input":"2023-06-09T17:35:18.629569Z","iopub.status.idle":"2023-06-09T17:35:18.656237Z","shell.execute_reply.started":"2023-06-09T17:35:18.629529Z","shell.execute_reply":"2023-06-09T17:35:18.655001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## deviceId","metadata":{}},{"cell_type":"code","source":"train_df.deviceId.value_counts().value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:18.657790Z","iopub.execute_input":"2023-06-09T17:35:18.658690Z","iopub.status.idle":"2023-06-09T17:35:18.673065Z","shell.execute_reply.started":"2023-06-09T17:35:18.658659Z","shell.execute_reply":"2023-06-09T17:35:18.671703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2, figsize=(15,10))\nsns.kdeplot(train_df.deviceId.value_counts(), ax=ax[0]).set(title='deviceId frequency')\nsns.kdeplot(train_df.deviceId.value_counts(), ax=ax[1], log_scale=True).set(title='deviceId frequency log scale');","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:18.674444Z","iopub.execute_input":"2023-06-09T17:35:18.674834Z","iopub.status.idle":"2023-06-09T17:35:20.056054Z","shell.execute_reply.started":"2023-06-09T17:35:18.674799Z","shell.execute_reply":"2023-06-09T17:35:20.054760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## winBid - target viariable","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2, figsize=(15,10))\nsns.kdeplot(train_df.winBid, ax=ax[0]).set(title='winBid distribution')\nsns.kdeplot(train_df.winBid, ax=ax[1], log_scale=True).set(title='winBid distribution log scale');","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:20.061719Z","iopub.execute_input":"2023-06-09T17:35:20.062145Z","iopub.status.idle":"2023-06-09T17:35:22.130109Z","shell.execute_reply.started":"2023-06-09T17:35:20.062113Z","shell.execute_reply":"2023-06-09T17:35:22.128855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## sentPrice","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=2, figsize=(15,10))\nsns.kdeplot(train_df.sentPrice, ax=ax[0]).set(title='sentPrice distribution')\nsns.kdeplot(train_df.sentPrice, ax=ax[1], log_scale=True).set(title='sentPrice distribution log scale');","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:22.131854Z","iopub.execute_input":"2023-06-09T17:35:22.132378Z","iopub.status.idle":"2023-06-09T17:35:24.023044Z","shell.execute_reply.started":"2023-06-09T17:35:22.132331Z","shell.execute_reply":"2023-06-09T17:35:24.021834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Categorical / level features","metadata":{}},{"cell_type":"code","source":"for col in ['unitDisplayType', 'brandName', 'bundleId', 'appVersion',\n       'correctModelName', 'countryCode', 'deviceId', 'osAndVersion',\n       'connectionType', 'c1', 'c2', 'c3', 'c4', 'size',\n       'mediationProviderVersion', 'bidFloorPrice', 'has_won']:\n    print(col)\n    print('unique values:', train_df[col].nunique())\n    print('distribution: \\n', train_df[col].value_counts())\n    print()\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:24.024421Z","iopub.execute_input":"2023-06-09T17:35:24.024742Z","iopub.status.idle":"2023-06-09T17:35:24.090909Z","shell.execute_reply.started":"2023-06-09T17:35:24.024715Z","shell.execute_reply":"2023-06-09T17:35:24.089655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Interactions","metadata":{}},{"cell_type":"code","source":"sns.scatterplot(x=train_df.winBid, y=train_df.sentPrice, alpha=0.01);","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:24.092706Z","iopub.execute_input":"2023-06-09T17:35:24.093461Z","iopub.status.idle":"2023-06-09T17:35:24.517531Z","shell.execute_reply.started":"2023-06-09T17:35:24.093418Z","shell.execute_reply":"2023-06-09T17:35:24.516324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[:, ['winBid', 'sentPrice']].corr()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:24.519274Z","iopub.execute_input":"2023-06-09T17:35:24.519725Z","iopub.status.idle":"2023-06-09T17:35:24.533932Z","shell.execute_reply.started":"2023-06-09T17:35:24.519686Z","shell.execute_reply":"2023-06-09T17:35:24.532534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['winBid_lagged'] = train_df.loc[:, ['deviceId', 'winBid']].groupby('deviceId').shift(1)['winBid']","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:24.535511Z","iopub.execute_input":"2023-06-09T17:35:24.536099Z","iopub.status.idle":"2023-06-09T17:35:24.557482Z","shell.execute_reply.started":"2023-06-09T17:35:24.536058Z","shell.execute_reply":"2023-06-09T17:35:24.556348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.scatterplot(x=train_df.winBid, y=train_df.winBid_lagged, alpha=0.01);","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:24.558933Z","iopub.execute_input":"2023-06-09T17:35:24.559382Z","iopub.status.idle":"2023-06-09T17:35:25.045965Z","shell.execute_reply.started":"2023-06-09T17:35:24.559343Z","shell.execute_reply":"2023-06-09T17:35:25.044660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[:, ['winBid', 'winBid_lagged']].corr()","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:25.047337Z","iopub.execute_input":"2023-06-09T17:35:25.047762Z","iopub.status.idle":"2023-06-09T17:35:25.061706Z","shell.execute_reply.started":"2023-06-09T17:35:25.047725Z","shell.execute_reply":"2023-06-09T17:35:25.060279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df, test_df","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:25.063736Z","iopub.execute_input":"2023-06-09T17:35:25.064221Z","iopub.status.idle":"2023-06-09T17:35:25.070217Z","shell.execute_reply.started":"2023-06-09T17:35:25.064182Z","shell.execute_reply":"2023-06-09T17:35:25.068966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineering\n\nIt's a `deviceId`-level time series problem, so to create features for test set I merged it with thre train set.\nThere is no `leak` of information from test set to train set. All features are engineered in time-aware fashion. It allows proper local validation.","metadata":{}},{"cell_type":"code","source":"train_df, test_df, sub = load_data(CONFIG)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:38:00.634638Z","iopub.execute_input":"2023-06-10T06:38:00.635172Z","iopub.status.idle":"2023-06-10T06:38:01.215148Z","shell.execute_reply.started":"2023-06-10T06:38:00.635131Z","shell.execute_reply":"2023-06-10T06:38:01.213319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, test_df, sub = load_data(CONFIG)\ntrain_df['test'] = 0\ntest_df['test'] = 1\n\ntrain_df = pd.concat([train_df, test_df])\ndel test_df\ntrain_df.sort_values(by=['deviceId', 'eventTimestamp'], inplace=True, ignore_index=True)\ntrain_df['lagged'] = train_df.loc[:, ['deviceId', 'eventTimestamp']].groupby('deviceId').rank(ascending=False)['eventTimestamp'].astype(np.int32) - 1\ntrain_df['cnt'] = train_df.loc[:, ['deviceId', 'eventTimestamp']].groupby('deviceId').rank()['eventTimestamp'].astype(np.int32)\ntrain_df['winBid_lagged'] = train_df.loc[:, ['deviceId', 'winBid']].groupby('deviceId').shift(1)['winBid']\ntrain_df['winBid_mean'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').cumsum()['winBid_lagged']\ntrain_df['winBid_min'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').cummin()['winBid_lagged']\ntrain_df['winBid_max'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').cummax()['winBid_lagged']\ntrain_df['winBid_mean'] = train_df['winBid_mean'] / train_df['cnt']\n\ntrain_df['winBid_roll_median'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').rolling(10).quantile(0.5)['winBid_lagged'].values\ntrain_df['winBid_roll_q1'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').rolling(10).quantile(0.25)['winBid_lagged'].values\ntrain_df['winBid_roll_q3'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').rolling(10).quantile(0.75)['winBid_lagged'].values\ntrain_df['winBid_roll_mean'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').rolling(10).mean()['winBid_lagged'].values\ntrain_df['winBid_roll_max'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').rolling(10).max()['winBid_lagged'].values\ntrain_df['winBid_roll_min'] = train_df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').rolling(10).min()['winBid_lagged'].values\n\ntrain_df['sentPrice_lagged'] = train_df.loc[:, ['deviceId', 'sentPrice']].groupby('deviceId').shift(1)['sentPrice']\ntrain_df['sentPrice_mean'] = train_df.loc[:, ['deviceId', 'sentPrice_lagged']].groupby('deviceId').cumsum()['sentPrice_lagged']\ntrain_df['sentPrice_min'] = train_df.loc[:, ['deviceId', 'sentPrice_lagged']].groupby('deviceId').cummin()['sentPrice_lagged']\ntrain_df['sentPrice_max'] = train_df.loc[:, ['deviceId', 'sentPrice_lagged']].groupby('deviceId').cummax()['sentPrice_lagged']\ntrain_df['sentPrice_mean'] = train_df['sentPrice_mean'] / train_df['cnt']\n\ntrain_df['sent_per_win'] = train_df['sentPrice_lagged'] / train_df['winBid_lagged']\ntrain_df['sent_per_win_mean'] = train_df['sentPrice_mean'] / train_df['winBid_mean']\ntrain_df['sent_per_win_min'] = train_df['sentPrice_min'] / train_df['winBid_min']\ntrain_df['sent_per_win_max'] = train_df['sentPrice_max'] / train_df['winBid_max']\n\ntrain_df['has_won_lagged'] = train_df.loc[:, ['deviceId', 'has_won']].groupby('deviceId').shift(1)['has_won']\ntrain_df['has_won_mean'] = train_df.loc[:, ['deviceId', 'has_won_lagged']].groupby('deviceId').cumsum()['has_won_lagged']\ntrain_df['has_won_min'] = train_df.loc[:, ['deviceId', 'has_won_lagged']].groupby('deviceId').cummin()['has_won_lagged']\ntrain_df['has_won_max'] = train_df.loc[:, ['deviceId', 'has_won_lagged']].groupby('deviceId').cummax()['has_won_lagged']\ntrain_df['has_won_mean'] = train_df['has_won_mean'] / train_df['cnt']\n\ntrain_df['eventTimestamp_lagged'] = train_df.loc[:, ['deviceId', 'eventTimestamp']].groupby('deviceId').shift(1)['eventTimestamp']\ntrain_df['time_to_last_event'] = train_df.eventTimestamp - train_df.eventTimestamp_lagged\n\ntrain_df['bundleId_lagged'] = train_df.loc[:, ['deviceId', 'bundleId']].groupby('deviceId').shift(1)['bundleId']\n\ntrain_df['eventTimestamp2'] = (train_df.eventTimestamp * 1_000_000).astype('datetime64[ns]')\ntrain_df['eventTimestamp2_lagged'] = (train_df.eventTimestamp * 1_000_000).astype('datetime64[ns]')\ntrain_df['month_day'] = train_df.eventTimestamp2.dt.day\ntrain_df['weekday'] = train_df.eventTimestamp2.dt.weekday\ntrain_df['month'] = train_df.eventTimestamp2.dt.month\ntrain_df['hour'] = train_df.eventTimestamp2.dt.hour\n\ntrain_df['cc'] = train_df.c2 / train_df.c4\n\ntrain_df['date_bin'] = train_df['month'] * 10000 + train_df['month_day'] * 100 + train_df['hour']\n\ntrain_df.drop(['eventTimestamp2', 'eventTimestamp2_lagged', 'eventTimestamp'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:25.320201Z","iopub.execute_input":"2023-06-09T17:35:25.320555Z","iopub.status.idle":"2023-06-09T17:35:45.710395Z","shell.execute_reply.started":"2023-06-09T17:35:25.320527Z","shell.execute_reply":"2023-06-09T17:35:45.709175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation schema\n\nI tried to reproduce the way submission is done - forecast one event for every `deviceId` ahead. There is `GroupKFold` split done on the train set, but it's used for training only. The validation is done on the \"future\" validation events.\n\n![validation schema.png](attachment:67f40155-774f-4e18-8811-2787e2f18f3c.png)","metadata":{},"attachments":{"67f40155-774f-4e18-8811-2787e2f18f3c.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAvYAAADQCAYAAABsiyUiAAATqElEQVR4Xu3da4yda1XA8Xfvmban5chFQY3R5GiiRkWjwQ+QGOVo1KAkXIIkSoQE1A+o4aaJEAOCaFQMgne8QBAELxhDFPCDlxA1CNoUUxGLBk5qSrQcQCggHkrrs7Lf2nE905m9Zk9nePf+/ZJ/OrN3n8XMTuZ9F5M5nWEAAAAAAADWwMmTp660P65LkiTp8Nra2uoeq3Tq1KnusWqz2ezaMF33Drt8TpXm8/mHhg1z/dzFK5IkSTrEYse6cPn8gYvzVy+dXamYkfa+Kele02oxIw9dd92LIEmSpNWKHSsv65XifF7Uq8WMtPdNSfeaVosZeei6614ESZIkrVbsWHlZrxTn86JeLWakvW9Kute0WszIQ3fx6Nar9uhFN/9qybe3fjo/eLt1L4IkSZJWK3asvKxXivN5Ua8WM9LeNyXda1otZuShu9i52P/OsDjz1h2PHXSx/77Wn+cHb7fuRZAkSdJqxY6Vl/VKcT4v6tViRtr7pqR7TavFjDx0H3cMizNPyk9MRfciSJIkabVix8rLeqU4nxf1ajEj7X1T0r2m1WJGHrqPWy328X585/6XW+9tPbk1az2j9fet97X+qfXi1vZ45omtV49vf1nrza27W387LP7+H7fuGp8/NN2LIEmSpNWKHSsv65XifF7Uq8WMtPdNSfeaVosZeeg+brXYP7f10dZbxrcf3nrOsPgnOZ/Q+qrWD7Tuaz1tPPPs1rvHtx/W+lTrXOvxrUeNzx36j+p0L4IkSZJWK3asvKxXivN5Ua8WM9LeNyXda1otZuSh+9hrsf9E68yOx76n9dgd74f4bvxLxrfzYh9zv2l8Pzy19eEd7x+K7kWQJEnSasWOlZf1SnE+L+rVYkba+6ake02rxYw8dB97LfbxozbZ17V+pPWK1jtaV1svHZ/bbbH/vPH9EN+5//iO9w9F9yJIkiRptWLHyst6pTifF/VqMSPtfVPSvabVYkYeuo+9Fvuz6bGfaH2y9XvD4sdy4ufn3zbsvdg/eHw/PG6w2EuSJH3mFztWXtYrxfm8qFeLGWnvm5LuNa0WM/LQfSy72Md/OBtLeXy3fudj97ReNr5vsZckSVqHYsfKy3qlOJ8X9WoxI+19U9K9ptViRh66j2UX+/D+1u+37mw9pPWLrU8Pix/LCRZ7SZKkdSh2rLysV4rzeVGvFjPS3jcl3WtaLWbkofs4NSz+Kcr8H8U+vfUn6bFvab1nWPxvXGo9v/Vjw+KftQw7f0HVQ4fF3AeN74f4zbT/vOP9Q9G9CJIkSVqt2LHysl4pzudFvVrMSHvflHSvabWYkYeuu+5FkCRJ0mrFjpWX9UpxPi/q1WJG2vumpHtNq8WMPHStnbrjdPxsz41PXJIkSYfQ9vZ291il06fv6B6rNpvNrg3TFb/8qfucKs3n8w8NAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwCY4febMfw+7/NpzSevXiRPb3WOVztzvTPdYtdlsdm2YrnuHXT6nSltbWx8eAOA2uX7u4hVJG1B8vV+4fP7Axfmrl86uVMxI16Ap6V6TajEjDwWAw9Ld/CWtZ/H1nhfNSnE+L+rVYka6Bk1J95pUixl5KAAclu7mL2k9i6/3vGhWivN5Ua8WM9I1aEq616RazMhDd/Gi1ttas/zE6PWtl+UHkwe23tf66vH9N7aecfPpzitbL8gPAjAt3c1f0noWX+950awU5/OiXi1mpGvQlHSvSbWYkYfu4hHD4u99Q36i+cLW1da35SeSzxkWM75+fP/5rcfcfLrzp62X5wcBmJbu5i9pPYuv97xoVorzeVGvFjPSNWhKutekWszIQ2/hXa1fyw82z23d05qP78d39e8au/FYyIv9bh7U+qLxbYs9wBrobv6S1rP4es+LZqU4nxf1ajEjXYOmpHtNqsWMPPQWnt36YOtkevxfhsV338PTh8Xfif5n/PMJ43N5sX/HcPNHbWLmbw2L7/zHmbePWewBJq67+Utaz+LrPS+aleJ8XtSrxYx0DZqS7jWpFjPy0Ft4yLBY1nf++MzDh8UyHt9lf2jr063Hjs/Fsv661nvH9/da7H+8dan1FeP7Tx0Wf9diDzBx3c1f0noWX+950awU5/OiXi1mpGvQlHSvSbWYkYfu4Q2tP9zxfvxozpvGtz+7dfeO58JzWh8b395rsY/v+j9vfPsG37EHWAPdzV/SehZf73nRrBTn86JeLWaka9CUdK9JtZiRh+7hUa34JYIPaN3Ril9u9bgdzz+y9ZrWP7Q+2vpA6xPjc3st9vcNN39k54ZXDRZ7gMnrbv6S1rP4es+LZqU4nxf1ajEjXYOmpHtNqsWMPHQPW62Lrae1ntj6j9b2+Fws/fFjOS8Z3/7c1vcOyy32H2k9eXz7htcOFnuAyetu/pLWs/h6z4tmpTifF/VqMSNdg6ake02qxYw8dB8/2XpL649aP7Pj8V9tvXPH++GXWp8c395rsY8f5/nd8e0QP59/z2CxB5i87uYvaT2Lr/e8aFaK83lRrxYz0jVoSrrXpFrMyEP38cXD4sdxPt760h2Pf/+w+JGaZ7a+tfXSYfEfxMb8M8Pei308Ft/Z//XWk1p/OSy+i2+xB5i47uYvaT2Lr/e8aFaK83lRrxYz0jVoSrrXpFrMyEOX8MJh8Z37neLfr//h1p8Ni+/oP6v14GHxs/Jf2bpzfPuu8e/nX1D1ta1Xt/5iWMyJ/6MQSz4AE9bd/CWtZ/H1nhfNSnE+L+rVYka6Bk1J95pUixl5KAAclu7mL2k9i6/3vGhWivN5Ua8WM9I1aEq616RazMhDAeBQnDhxMv6JtBs3G0lr3Hw+7x6rdOLkie6xA3RtmK57h/7zKTWbzeI3vQIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALey8m8Nk6Q1K66LAOztMHZI19tDdv3cxSuS9BlRXJMuXD5/4OL81UtnVypmpOskAL3u+lktZuShrKa7sUrScRXXpLysV4rz+cZRLWak6yQAve76WS1m5KGspruxStJxFdekvKxXivP5xlEtZqTrJAC97vpZLWbkobt4QOuuPfqCgf/T3Vgl6biKa1Je1ivF+XzjqBYz0nUSgF53/awWM/LQXTx3WPy9W/WPN//qgXxX6wn5wanqbqySdFzFNSkv65XifL5xVIsZ6ToJQK+7flaLGXnoLnZ+x/7Lh8WZZ+54bNXv2F9o/VB+cKq6G6skHVdxTcrLeqU4n28c1WJGuk4C0Ouun9ViRh66jzuGxZkn5Seau1tPaX1Hazs9F8t/fGc+nv+aHY8/rHWx9fL0+GR1N1ZJOq7impSX9UpxPt84qsWMdJ0EoNddP6vFjDx0H7st9vdr/VXr/a3Xt97demfrwePz39i60npz6w9aH2v9/Pjcb7Q+3npP62fHxyatu7FK0nEV16S8rFeK8/nGUS1mpOskAL3u+lktZuSh+9htsf+51r+1Hji+H9+t/+vWK8f3X9d61fh2eHTrb1onxvf9KI4k3Y7impSX9UpxPt84qsWMdJ0EoNddP6vFjDx0H7st9v/aem3rkTv6hdYHxudf3PpI64Wth7dm4+M3WOwl6XYU16S8rFeK8/nGUS1mpOskAL3u+lktZuSh+9htsY8frYkl/n27tDUszvxU655hcfY/W8+LgyOLvSTdjuKalJf1SnE+3ziqxYx0nQSg110/q8WMPHQfuy32/9760R3vhzuHm9+Znw+Ln8MP8a/qxHfwY8Y3j49Z7CXpdhTXpLysV4rz+cZRLWak6yQAve76WS1m5KH72G2x/+3Wu4bFMn/DG4bFf0Ab3t76lR3P3b/1qdZjxvfj7LNvPj1t3Y1Vko6ruCblZb1SnM83jmoxI10nAeh1189qMSMP3cdui/3nD4vvut/T+s3WW4fFz9Q/Ynz+O1v3td40LP4VnPhXc/6udWp8/o2t/2q9Ynx/0robqyQdV3FNyst6pTifbxzVYka6TgLQ666f1WJGHrqP+Jn5p7S+JD0eS/p3t17Q+sFhsezvFD+C86xh8fzjh///79w/aFj8H4W1+O2z3Y1Vko6ruCblZb1SnM83jmoxI10nAeh1189qMSMPZTXdjVWSjqu4JuVlvVKczzeOajEjXScB6HXXz2oxIw9lBbPZ7IPD4kWVpGOvXZO6xyrN5/PusQN07wDAfuJama+f1VxvAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAFfkNvZIkHW6H8Jumrw0AB3D93MUrkiSpFffFC5fPr1TMuHrp7IGL8+leDbCU7qImSdKmFvfFvKhXixl5Wa8U59O9GmAp3UVNkqRNLe6LeVGvFjPysl4pzqd7NcBSuouaJEmbWtwX86JeLWbkZb1SnE/3aoCldBc1SZI2tbgv5kW9WszIy3qlOJ/u1QBL6S5qkiRtanFfzIt6tZiRl/VKcT7dqwGW0l3UJEna1OK+mBf1ajEjL+uV4ny6VwMspbuoSZK0qcV9MS/q1WJGXtYrxfl0rwZYSndRkyRpU4v7Yl7Uq8WMvKxXivPpXg2wlO6iJknSphb3xbyoV4sZeVmvFOfTvRpgKd1FTZKkTS3ui3lRrxYz8rJeKc6nezXAUrqLmiRJm1rcF/OiXi1m5GW9UpxP92qApXQXNUmSNrW4L+ZFvVrMyMt6pTif7tUAS+kuapIkbWpxX8yLerWYkZf1SnE+3asBltJd1CRJ2tTivpgX9WoxIy/rleJ8ulcD7G97+8RHhsUFRJKkjW82m3WPVdva2uoeq3Tq5MlrAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA62Q+m11rf1xfpdliBgAAcIyuX710dqViRh4KAAAcrW5RrxYz8lAAAOBodYt6tZiRhwIAAEerW9SrxYw8FAAAOFrdol4tZuShAADA0eoW9WoxIw8FAACOVreoV4sZeSgAAHC0ukW9WszIQwEAgKPVLerVYkYeCgAAHK1uUa8WM/JQAADgaHWLerWYkYcCAABHq1vUq8WMPBQAADha3aJeLWbkoQAAwNHqFvVqMSMPBQAAjla3qFeLGXkoAABwhObz+bVhsZgfuNlsFjMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgM8MJ0+dvNb+uH7QTp853T1W6f533i/+BAAAVnT9wuXzBy7OX7109sDF+fwBAQAAdd2yXinO52W9UpzPHxAAAFDXLeuV4nxe1ivF+fwBAQAAdd2yXinO52W9UpzPHxAAAFDXLeuV4nxe1ivF+fwBAQAAdd2yXinO52W9UpzPHxAAAFDXLeuV4nxe1ivF+fwBAQAAdd2yXinO52W9UpzPHxAAAFDXLeuV4nxe1ivF+fwBAQAAdd2yXinO52W9UpzPHxAAAFDXLeuV4nxe1ivF+fwBAQAAdd2yXinO52W9UpzPHxAAAFDXLeuV4nxe1ivF+fwBAQAAdd2yXinO52W9UpzPHxAAAFDXLeuV4nxe1ivF+fwBAQAARVtbW9eGxXJ9oLZPbHePVTp18mT87wMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEzbvcMuv7FNkiQdS35jKnBg189dvCJJklpxX7xw+fyBi/NXL509cHE+3acBltZd1CRJ2tTivpiX9UpxPi/rleJ8uk8DLK27qEmStKnFfTEv65XifF7WK8X5dJ8GWFp3UZMkaVOL+2Je1ivF+bysV4rz6T4NsLTuoiZJ0qYW98W8rFeK83lZrxTn030aYGndRU2SpE0t7ot5Wa8U5/OyXinOp/s0wNK6i5okSZta3Bfzsl4pzudlvVKcT/dpgKV1FzVJkja1uC/mZb1SnM/LeqU4n+7TAEvrLmqSJG1qcV/My3qlOJ+X9UpxPt2nAZbWXdQkSdrU4r6Yl/VKcT4v65XifLpPAyytu6hJkrSpxX0xL+uV4nxe1ivF+XSfBlhad1GTJGlTi/tiXtYrxfm8rFeK8+k+DbC07qImSdKmFvfFvKxXivN5Wa8U59N9GmBp3UVNkqRNLe6LeVmvFOfzsl4pzqf7NMDSuouaJEmbWtwX87JeKc7nZb1SnE/3aYCldRc1SZI2tbgv5mW9UpzPy3qlOJ/u0wDLmc/nHxoWFxFJkja+2XzWPVZpNlvt/Hw+uzYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC3wXxr68PDLr9deNlW/S3I29tb3WMHyG9CBgBg410/d/HKgYvzFy6fP3Bx/uqlsysVM9LnBAAAG6db1ivF+bysV4rzeVGvFjPS5wQAABunW9Yrxfm8rFeK83lRrxYz0ucEAAAbp1vWK8X5vKxXivN5Ua8WM9LnBAAAG6db1ivF+bysV4rzeVGvFjPS5wQAABunW9Yrxfm8rFeK83lRrxYz0ucEAAAbp1vWK8X5vKxXivN5Ua8WM9LnBAAAG6db1ivF+bysV4rzeVGvFjPS5wQAABunW9Yrxfm8rFeK83lRrxYz0ucEAAAbp1vWK8X5vKxXivN5Ua8WM9LnBAAAG6db1ivF+bysV4rzeVGvFjPS5wQAABunW9Yrxfm8rFeK83lRrxYz0ucEAAAbp1vWK8X5vKxXivN5Ua8WM9LnBAAAG6db1ivF+bysV4rzeVGvFjPS5wQAABunW9Yrxfm8rFeK83lRrxYz0ucEAACb5Y7TZz4xLBbjA3Xi5InusUp3ftad3WPVZrPZtQEAAAAAAICC/wX7MMJYmsPyUQAAAABJRU5ErkJggg=="}}},{"cell_type":"markdown","source":"# Model training\n\nMain points:\n* Categorical features:\n    * small categories and `nan` are merged into `__OTHER__`\n    * mapped into numbers with `OrdinalEncoder` (like `LabelEncoder` in `scikit-learn`)\n    * above operations are done on the current fold train set\n    * encoded features are passed to `categorical_features` in `LightGBM` (Exclusive Feature Bundling + other optimizations)\n* Target variable\n    * clipped to `99.99` quantile\n* Model params\n    * `rmse` objective\n    * `dart` boosting type for better accuracy\n    * `500` iterations for time\n    * optimized with `optuna` and `prunning` technique (see Appendix)\n* Predictions postprocessing\n    * mean from fold predictions\n    * clipped from `0.01`","metadata":{}},{"cell_type":"code","source":"class MergeSmallCats(TransformerMixin):\n    '''\n    merges small classess into `small_cat`\n    '''\n    def __init__(self, min_cnt=100, small_cat='__OTHER__'):\n        self.min_cnt = min_cnt\n        self.small_cat = small_cat\n        self.mapper = None\n\n    def fit(self, x):\n        cats = x.value_counts()\n        cats = cats[cats > self.min_cnt]\n        self.mapper = {i:i for i in cats.index}\n        return self\n\n    def transform(self, x):\n        x = x.map(self.mapper)\n        x = x.fillna(self.small_cat)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:45.711820Z","iopub.execute_input":"2023-06-09T17:35:45.712425Z","iopub.status.idle":"2023-06-09T17:35:45.721135Z","shell.execute_reply.started":"2023-06-09T17:35:45.712393Z","shell.execute_reply":"2023-06-09T17:35:45.719735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# based on previous run\nfilter_features = ['has_won_min', 'month', 'has_won_lagged', 'has_won_max', 'c3', 'bidFloorPrice']\ntrain_df.drop(filter_features, axis=1, inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_features = [ 'unitDisplayType', 'brandName', 'bundleId',\n       'appVersion', 'correctModelName', 'countryCode', \n       'osAndVersion', 'connectionType', 'c1', 'size', 'mediationProviderVersion', 'bundleId_lagged']","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:45.722689Z","iopub.execute_input":"2023-06-09T17:35:45.723244Z","iopub.status.idle":"2023-06-09T17:35:45.732970Z","shell.execute_reply.started":"2023-06-09T17:35:45.723215Z","shell.execute_reply":"2023-06-09T17:35:45.732067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cross_val_predict(params, x, x_test_base, categorical_features, splits, n_folds=5, plot_errors=True):\n\n    x_test_base.drop(['winBid', 'lagged', 'deviceId'], axis=1, inplace=True)\n\n    for split in splits:\n        scores = []\n        test_preds = []\n        feat_imps = []\n        time_start = time.time()\n        x_valid_base = x.loc[x.lagged == split, :]\n        y_valid_base = x_valid_base.loc[:, 'winBid']\n        x_train_base = x.loc[x.lagged > split, :]\n        y_train_base = x_train_base.loc[:, 'winBid']\n        group_base = x_train_base.deviceId\n        gkf = GroupKFold(n_splits=n_folds)\n\n        x_valid_base.drop(['winBid', 'lagged', 'deviceId'], axis=1, inplace=True)\n        x_train_base.drop(['winBid', 'lagged', 'deviceId'], axis=1, inplace=True)\n\n\n        categorical_features_idx = []\n        for i, col in enumerate(x_valid_base.columns):\n            if col in categorical_features:\n                categorical_features_idx.append(i)\n\n        y_valid_hat_all = []\n        for train_idx, _ in gkf.split(x_train_base, y_train_base, group_base):\n\n            x_valid = deepcopy(x_valid_base)\n            y_valid = deepcopy(y_valid_base)\n            x_test = deepcopy(x_test_base)\n\n            time_start = time.time()\n            x_train = x_train_base.iloc[train_idx, :]\n            y_train = y_train_base.iloc[train_idx]\n\n            y_train_q = y_train.quantile(0.9999)\n            y_train = np.clip(y_train, None, y_train_q)\n\n            params['categorical_features'] = categorical_features_idx\n            model = LGBMRegressor(**params)\n\n            for feat in categorical_features:\n                msc = MergeSmallCats(min_cnt=991)\n                x_train[feat] = msc.fit_transform(x_train[feat])\n                x_valid[feat] = msc.transform(x_valid[feat])\n                x_test[feat] = msc.transform(x_test[feat])\n\n            oe = OrdinalEncoder()\n            x_train.iloc[:, categorical_features_idx] = oe.fit_transform(x_train.iloc[:, categorical_features_idx])\n            x_valid.iloc[:, categorical_features_idx] = oe.transform(x_valid.iloc[:, categorical_features_idx])\n\n            x_train = x_train.astype(np.float32)\n            x_valid = x_valid.astype(np.float32)\n\n            model.fit(x_train, y_train) \n            feat_imps.append(model.feature_importances_)\n            y_train_hat = model.predict(x_train)\n            y_train_hat = np.clip(y_train_hat, x_train.sentPrice, None)\n\n            rmse_train = np.sqrt(mean_squared_error(y_train, y_train_hat))\n\n            if split > 0:\n                y_valid_hat = model.predict(x_valid)\n                y_valid_hat_all.append(y_valid_hat)\n                y_valid_hat = np.clip(y_valid_hat, x_valid.sentPrice, None)\n                rmse_valid = np.sqrt(mean_squared_error(y_valid, y_valid_hat))\n                scores.append(rmse_valid)\n                print(rmse_train, rmse_valid, int(time.time() - time_start))\n            else:\n                print(rmse_train, int(time.time() - time_start))\n\n            if x_test is not None and split == 0:\n                x_test.iloc[:, categorical_features_idx] = oe.transform(x_test.iloc[:, categorical_features_idx])\n                x_test = x_test.astype(np.float32)\n                y_test_hat = model.predict(x_test)\n                y_test_hat = np.clip(y_test_hat, x_test.sentPrice, None)\n                test_preds.append(y_test_hat)\n    \n        if split > 0:\n            y_valid_hat_all = np.vstack(y_valid_hat_all).mean(0)\n            y_valid_hat_all = np.clip(y_valid_hat_all, x_valid.sentPrice, None)\n            rmse_valid_all = np.sqrt(mean_squared_error(y_valid_base, y_valid_hat_all))\n            print(f'SCORE FOR SPLIT {split}: {rmse_valid_all}')\n            if plot_errors:\n                sns.scatterplot(x=np.log(y_valid_base), y=np.log(y_valid_hat_all), alpha=0.1)\n\n        feat_imps.append(model.feature_name_)\n        if x_test is not None and split == 0:\n            test_preds = np.vstack(test_preds).mean(0)\n            test_preds = np.clip(test_preds, x_test.sentPrice, None)\n\n    return test_preds, feat_imps","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:45.734509Z","iopub.execute_input":"2023-06-09T17:35:45.735186Z","iopub.status.idle":"2023-06-09T17:35:45.759569Z","shell.execute_reply.started":"2023-06-09T17:35:45.735154Z","shell.execute_reply":"2023-06-09T17:35:45.758426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\ntest_preds = []\n\nx = train_df.loc[train_df.lagged >= 1, :]\nx.drop(['has_won', 'test'], axis=1, inplace=True)\nx_test = train_df.loc[train_df.lagged == 0, :]\nx_test.drop(['has_won', 'test'], axis=1, inplace=True)\n\ny_test_hat, feat_imps = cross_val_predict(CONFIG.params, x, x_test, categorical_features, [1, 0], n_folds=5)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:35:45.763665Z","iopub.execute_input":"2023-06-09T17:35:45.764039Z","iopub.status.idle":"2023-06-09T17:36:37.210448Z","shell.execute_reply.started":"2023-06-09T17:35:45.764011Z","shell.execute_reply":"2023-06-09T17:36:37.209232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imps = pd.DataFrame(feat_imps[:-1], columns=feat_imps[-1]).mean().sort_values(ascending=False).to_frame().reset_index()\nimps.columns = ['feature', 'importnace']\nsns.set(rc={'figure.figsize':(10, 12)})\nsns.barplot(data=imps, y='feature', x='importnace', orient='h');","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:36:37.212049Z","iopub.execute_input":"2023-06-09T17:36:37.213122Z","iopub.status.idle":"2023-06-09T17:36:38.212354Z","shell.execute_reply.started":"2023-06-09T17:36:37.213086Z","shell.execute_reply":"2023-06-09T17:36:38.211358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_hat_map = pd.Series(y_test_hat, index=train_df.loc[train_df.lagged == 0, 'deviceId'])\nsub.winBid = sub.deviceId.map(y_test_hat_map)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:36:38.213730Z","iopub.execute_input":"2023-06-09T17:36:38.214049Z","iopub.status.idle":"2023-06-09T17:36:38.425969Z","shell.execute_reply.started":"2023-06-09T17:36:38.214022Z","shell.execute_reply":"2023-06-09T17:36:38.425100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Next steps\n\n1. Switch to GPU\n1. Try to use other algorithms: `XGBoost`, `Catboost`, `HistGradientBoosting`\n1. Try new categorical features encodings: time-wise Target encoding, Binary encoding, etc\n1. More feature engineering:\n    1. Extract data from `CorrectModelName`\n    1. More numeric ineraction features","metadata":{}},{"cell_type":"markdown","source":"# Appendix: Hyperparameter optimization","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:37:08.344775Z","iopub.execute_input":"2023-06-09T17:37:08.345257Z","iopub.status.idle":"2023-06-09T17:37:08.350496Z","shell.execute_reply.started":"2023-06-09T17:37:08.345216Z","shell.execute_reply":"2023-06-09T17:37:08.349081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = train_df.loc[train_df.lagged >= 1, :]\nx.drop(['has_won', 'test'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:37:08.604022Z","iopub.execute_input":"2023-06-09T17:37:08.604546Z","iopub.status.idle":"2023-06-09T17:37:08.623086Z","shell.execute_reply.started":"2023-06-09T17:37:08.604506Z","shell.execute_reply":"2023-06-09T17:37:08.621834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective(trial):\n    _EPS = 1e-12\n    joblib.dump(study, 'study.pkl')\n    num_leaves = trial.suggest_int('num_leaves', 10, 512)\n    lambda_l1 = trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True)\n    lambda_l2 = trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True)\n    feature_fraction = min(trial.suggest_float(\"feature_fraction\", 0.4, 1.0 + _EPS), 1.0)\n    bagging_fraction = min(trial.suggest_float(\"bagging_fraction\", 0.4, 1.0 + _EPS), 1.0)\n    bagging_freq = trial.suggest_int(\"bagging_freq\", 1, 7)\n    min_child_samples = trial.suggest_int(\"min_child_samples\", 5, 100)\n    min_cnt = trial.suggest_int('min_cnt', 10, 5000)\n\n    params = {'lambda_l1': lambda_l1,\n              'lambda_l2': lambda_l2,\n              'num_leaves': num_leaves,\n              'feature_fraction': feature_fraction,\n              'bagging_fraction': bagging_fraction,\n              'bagging_freq': bagging_freq,\n              'min_child_samples': min_child_samples,\n              'num_iterations': 500,\n              'boosting_type': 'dart',\n              'objective': 'rmse',\n              'metric': 'rmse',\n              'verbose': -1\n              }\n\n    x_valid = x.loc[x.lagged == 1, :]\n    y_valid = x_valid.loc[:, 'winBid']\n    x_train = x.loc[x.lagged > 1, :]\n    x_train = x_train.sample(frac=0.8, random_state=42)\n    y_train = x_train.loc[:, 'winBid']\n\n    x_valid.drop(['winBid', 'lagged', 'deviceId'], axis=1, inplace=True)\n    x_train.drop(['winBid', 'lagged', 'deviceId'], axis=1, inplace=True)\n\n    categorical_features_idx = []\n    for i, col in enumerate(x_train.columns):\n        if col in categorical_features:\n            categorical_features_idx.append(i)\n\n    params['categorical_features'] = categorical_features_idx\n\n    y_train_q = y_train.quantile(.9999)\n    y_train = np.clip(y_train, None, y_train_q)\n\n    params['categorical_features'] = categorical_features_idx\n\n    for feat in categorical_features:\n        msc = MergeSmallCats(min_cnt=min_cnt)\n        x_train[feat] = msc.fit_transform(x_train[feat])\n        x_valid[feat] = msc.transform(x_valid[feat])\n\n    oe = OrdinalEncoder()\n    x_train.iloc[:, categorical_features_idx] = oe.fit_transform(x_train.iloc[:, categorical_features_idx])\n    x_valid.iloc[:, categorical_features_idx] = oe.transform(x_valid.iloc[:, categorical_features_idx])\n\n    x_train = x_train.astype(np.float32)\n    x_valid = x_valid.astype(np.float32)\n\n    dtrain = lgb.Dataset(x_train, label=y_train)\n    dvalid = lgb.Dataset(x_valid, label=y_valid)\n\n    # Add a callback for pruning.\n    pruning_callback = optuna.integration.LightGBMPruningCallback(trial, \"rmse\")\n    gbm = lgb.train(params, dtrain, valid_sets=[dvalid], callbacks=[pruning_callback])\n\n    y_valid_hat = gbm.predict(x_valid)\n    y_valid_hat = np.clip(y_valid_hat, 0.01, None)\n    return np.sqrt(mean_squared_error(y_valid, y_valid_hat))\n","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:39:30.942054Z","iopub.execute_input":"2023-06-09T17:39:30.942992Z","iopub.status.idle":"2023-06-09T17:39:30.961756Z","shell.execute_reply.started":"2023-06-09T17:39:30.942955Z","shell.execute_reply":"2023-06-09T17:39:30.960595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# study = optuna.create_study(pruner=optuna.pruners.MedianPruner(n_warmup_steps=10))\n# study.optimize(objective, n_trials=1000)\n# joblib.dump(study, 'study.pkl')","metadata":{"execution":{"iopub.status.busy":"2023-06-09T17:39:31.631290Z","iopub.execute_input":"2023-06-09T17:39:31.632124Z","iopub.status.idle":"2023-06-09T17:39:57.129361Z","shell.execute_reply.started":"2023-06-09T17:39:31.632091Z","shell.execute_reply":"2023-06-09T17:39:57.127519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}