{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Loading packages","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport time\nimport gc\nfrom datetime import datetime\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom math import sqrt\nimport lightgbm as lgb\n\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-17T12:00:28.415157Z","iopub.execute_input":"2022-09-17T12:00:28.415644Z","iopub.status.idle":"2022-09-17T12:00:30.489761Z","shell.execute_reply.started":"2022-09-17T12:00:28.415552Z","shell.execute_reply":"2022-09-17T12:00:30.488455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading raw data: train, test, sample submission","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/train_data.csv')\ntest = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/test_data.csv')\nsample = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/sample_submission.csv')\n\nprint(f'Number of rows in train set: {train.shape[0]}')\nprint(f'Number of rows in test set: {test.shape[0]}')\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:00:30.492518Z","iopub.execute_input":"2022-09-17T12:00:30.493254Z","iopub.status.idle":"2022-09-17T12:01:04.549984Z","shell.execute_reply.started":"2022-09-17T12:00:30.493192Z","shell.execute_reply":"2022-09-17T12:01:04.548723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preapre function to create features based on raw data (input train or test set)","metadata":{}},{"cell_type":"code","source":"brand_train_popularity = train['brandName'].value_counts().rename('popularity_of_brand')\napp_train_popularity = train['bundleId'].value_counts().rename('popularity_of_app')\ndevice_train_popularity = train['deviceId'].value_counts().rename('popularity_of_device')\n\ntrain['eventDate'] = pd.to_datetime(train['eventTimestamp'], unit='ms')\ntrain['hour'] = train['eventDate'].dt.hour\nmean_hour_device = train.groupby('deviceId')['hour'].mean()\n\nmean_win_bid_brand = train.groupby('brandName')['winBid'].mean()\nmean_win_bid_app = train.groupby('bundleId')['winBid'].mean()\n\nmean_win_bid_device = train.groupby('deviceId')['winBid'].mean()\nmedian_win_bid_device = train.groupby('deviceId')['winBid'].median()\nstd_win_bid_device = train.groupby('deviceId')['winBid'].std()\nmin_win_bid_device = train.groupby('deviceId')['winBid'].min()\nmax_win_bid_device = train.groupby('deviceId')['winBid'].max()\n\nmean_sent_price_device = train.groupby('deviceId')['sentPrice'].mean()\nmedian_sent_price_device = train.groupby('deviceId')['sentPrice'].median()\nstd_sent_price_device = train.groupby('deviceId')['sentPrice'].std()\nmin_sent_price_device = train.groupby('deviceId')['sentPrice'].min()\nmax_sent_price_device = train.groupby('deviceId')['sentPrice'].max()\n\nmean_win_bid_c1 = train.groupby('c1')['winBid'].mean()\nmean_win_bid_c3 = train.groupby('c3')['winBid'].mean()\n\ndef create_features(df):\n    \n    df['eventDate'] = pd.to_datetime(df['eventTimestamp'], unit='ms')\n    df['hour'] = df['eventDate'].dt.hour\n    df['day_of_week'] = df['eventDate'].dt.dayofweek - 27\n    df['is_weekend'] = np.where(df['eventDate'].dt.dayofweek>4, 1, 0)\n    df['week_of_year'] = df['eventDate'].dt.week\n    df['mean_hour_device'] = df['deviceId'].map(mean_hour_device)\n    \n    df['is_banner'] = np.where(df['unitDisplayType'] == 'banner', 1, 0)\n    df['is_interstitial'] = np.where(df['unitDisplayType'] == 'interstitial', 1, 0)\n    df['is_rewarded'] = np.where(df['unitDisplayType'] == 'rewarded', 1, 0)\n    \n    df['popularity_of_brand'] = df['brandName'].map(brand_train_popularity)\n    df['popularity_of_app'] = df['bundleId'].map(app_train_popularity)\n    df['popularity_of_device'] = df['deviceId'].map(device_train_popularity)\n    \n    df['mean_win_bid_brand'] = df['deviceId'].map(mean_win_bid_brand)\n    df['mean_win_bid_app'] = df['deviceId'].map(mean_win_bid_app)\n    \n    df['is_apple'] = np.where(df['brandName'] == 'Apple', 1, 0)\n    df['is_android'] = np.where(df['osAndVersion'].str.split('-').str[0] == \"Android\", 1, 0)\n    df['is_ios'] = np.where(df['osAndVersion'].str.split('-').str[0] == \"iOS\", 1, 0)\n    \n    df['is_US'] = np.where(df['countryCode'] == 'US', 1, 0)\n    df['is_JP'] = np.where(df['countryCode'] == 'JP', 1, 0)\n    df['is_CH'] = np.where(df['countryCode'] == 'CH', 1, 0)\n    df['is_BR'] = np.where(df['countryCode'] == 'BR', 1, 0)\n    \n    df['mean_win_bid_device'] = df['deviceId'].map(mean_win_bid_device)\n    df['median_win_bid_device'] = df['deviceId'].map(median_win_bid_device)\n    df['std_win_bid_device'] = df['deviceId'].map(std_win_bid_device)\n    df['min_win_bid_device'] = df['deviceId'].map(min_win_bid_device)\n    df['max_win_bid_device'] = df['deviceId'].map(max_win_bid_device)\n    \n    df['mean_sent_price_device'] = df['deviceId'].map(mean_sent_price_device)\n    df['median_sent_price_device'] = df['deviceId'].map(median_sent_price_device)\n    df['std_sent_price_device'] = df['deviceId'].map(std_sent_price_device)\n    df['min_sent_price_device'] = df['deviceId'].map(min_sent_price_device)\n    df['max_sent_price_device'] = df['deviceId'].map(max_sent_price_device)\n    \n    df['is_WIFI'] = np.where(df['connectionType'] == 'WIFI', 1, 0)\n    df['is_3G'] = np.where(df['connectionType'] == '3G', 1, 0)\n    \n    df['mean_win_bid_c1'] = df['c1'].map(mean_win_bid_c1)\n    df['mean_win_bid_c3'] = df['c3'].map(mean_win_bid_c3)\n    \n    df['c2_multiply_c4'] = df['c2'] * df['c4']\n    \n    df['size_width'] = df['size'].str.split('x').str[0].astype(int)\n    df['size_height'] = df['size'].str.split('x').str[1].astype(int)\n    df['size_pixels'] = df['size_width'] * df['size_height']\n    \n    df['mediation_minor'] = df['mediationProviderVersion'].str.split('.').str[1].astype(int)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:01:04.551791Z","iopub.execute_input":"2022-09-17T12:01:04.552422Z","iopub.status.idle":"2022-09-17T12:01:21.032794Z","shell.execute_reply.started":"2022-09-17T12:01:04.552384Z","shell.execute_reply":"2022-09-17T12:01:21.031778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Apply function and remove raw set","metadata":{}},{"cell_type":"code","source":"train_feats = create_features(train)\ntest_feats = create_features(test)\n\ndel train, test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:01:21.035134Z","iopub.execute_input":"2022-09-17T12:01:21.035502Z","iopub.status.idle":"2022-09-17T12:03:00.231519Z","shell.execute_reply.started":"2022-09-17T12:01:21.035471Z","shell.execute_reply":"2022-09-17T12:03:00.230136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create list of final features which will take part in creating model","metadata":{}},{"cell_type":"code","source":"FEATURES = ['hour', 'day_of_week', 'week_of_year', 'mean_hour_device', 'is_banner', 'is_interstitial', 'is_rewarded', 'popularity_of_brand', 'popularity_of_device', \n            'is_apple', 'is_US', 'mean_win_bid_device', 'median_win_bid_device', 'min_win_bid_device', 'max_win_bid_device', \n            'mean_sent_price_device', 'median_sent_price_device', 'min_sent_price_device', 'max_sent_price_device', 'is_WIFI',\n            'is_3G', 'mean_win_bid_c1', 'mean_win_bid_c3', 'size_width', 'size_height', 'mediation_minor', 'sentPrice']","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:03:00.232826Z","iopub.execute_input":"2022-09-17T12:03:00.233188Z","iopub.status.idle":"2022-09-17T12:03:00.239921Z","shell.execute_reply.started":"2022-09-17T12:03:00.233159Z","shell.execute_reply":"2022-09-17T12:03:00.238666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create list empty lists for folds and divide to n folds","metadata":{}},{"cell_type":"code","source":"models = []\nimportances = []\npredictions = []\noofs = []\n\nfolds = KFold(n_splits = 7, shuffle = True, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:03:00.241254Z","iopub.execute_input":"2022-09-17T12:03:00.241587Z","iopub.status.idle":"2022-09-17T12:03:00.252627Z","shell.execute_reply.started":"2022-09-17T12:03:00.241558Z","shell.execute_reply":"2022-09-17T12:03:00.251210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learn using params in folds loop and save results into lists","metadata":{}},{"cell_type":"code","source":"params = {\n    'task': 'train',\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': 'rmse',\n    'learning_rate': 0.1,\n    #'colsample_bytree': 0.2,\n    'bagging_fraction': 0.6,\n    #'bagging_freq': 10,\n    'verbose': -1,\n    \"max_depth\": 12,\n    \"n_estimators\": 3200\n}\n\nfor fold, (train_idx, valid_idx) in enumerate(folds.split(train_feats, train_feats['winBid'])):\n\n    start = time.time()\n    print(f'--------- Fold: {fold} ----------')\n    \n    train_feats['winBid']\n    X_train, y_train = train_feats[FEATURES].iloc[train_idx], train_feats['winBid'].iloc[train_idx]\n    X_valid, y_valid = train_feats[FEATURES].iloc[valid_idx],  train_feats['winBid'].iloc[valid_idx]\n\n    model = lgb.LGBMRegressor(**params)\n\n    model.fit(X_train[FEATURES], y_train, eval_set = (X_valid[FEATURES], y_valid),\n          callbacks = [lgb.early_stopping(40), lgb.log_evaluation(100)])\n    \n    oof = model.predict(X_valid[FEATURES], num_iteration = model.best_iteration_)\n    fold_score = sqrt(mean_squared_error(oof, y_valid))\n    print(f'RMSE OFF of this fold is: {fold_score:0.4f}')\n    \n    importance = pd.DataFrame(index = model.feature_name_, data = model.feature_importances_, columns = [f'{fold}_importance'])\n\n    fold_test_pred = model.predict(test_feats[FEATURES], num_iteration = model.best_iteration_)\n    \n    predictions.append(fold_test_pred)\n    models.append(model)\n    oofs.append(fold_score)\n    importances.append(importance)\n    \n    end = time.time()\n    full_time = (end - start)\n    \n    print(f'Time to train, predict and save results: {full_time:0.2f} seconds')","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:03:00.254536Z","iopub.execute_input":"2022-09-17T12:03:00.254984Z","iopub.status.idle":"2022-09-17T12:25:59.215797Z","shell.execute_reply.started":"2022-09-17T12:03:00.254943Z","shell.execute_reply":"2022-09-17T12:25:59.213770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Print general OOF (mean oof of all folds)","metadata":{}},{"cell_type":"code","source":"print(f'General CV (OOF): {np.mean(oofs):0.4f}')","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:25:59.217051Z","iopub.status.idle":"2022-09-17T12:25:59.217490Z","shell.execute_reply.started":"2022-09-17T12:25:59.217288Z","shell.execute_reply":"2022-09-17T12:25:59.217309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.float_format', lambda x: '%.3f' % x)\nimportances_df = pd.concat(importances, axis = 1)\nimportances_df['Feature'] = importance.index\nimportances_df['Value'] = importances_df['0_importance'] + importances_df['1_importance'] + importances_df['2_importance'] + importances_df['3_importance'] + \\\n                          importances_df['4_importance'] + importances_df['5_importance'] + importances_df['6_importance'] \nimportances_df = importances_df.sort_values('Value', ascending = False)\nimportances_df['Value'] = importances_df['Value'] / importances_df['Value'].sum()\n\nplt.rcParams['figure.dpi'] = 600\nfig = plt.figure(figsize = (5, 3), facecolor = 'white')\ngs = fig.add_gridspec(1, 2)\ngs.update(wspace = 0.3, hspace = 0)\nax0 = fig.add_subplot(gs[0, 0])\nax0.set_facecolor(\"white\")\nfor s in [\"right\", \"top\"]:\n    ax0.spines[s].set_visible(False)\n\nax0.text(0, -1.8, 'Feature importance ', color = 'black', fontsize = 7.3, ha = 'left', va = 'bottom', weight = 'bold')\nax0.text(0, -1.8, 'final lightGBM model', color = 'black', fontsize = 6, ha = 'left', va = 'top')\nax0_sns = sns.barplot(ax = ax0, y = importances_df['Feature'], x = importances_df['Value'], edgecolor = \"black\", linewidth = 0.3, color = \"#225478\", orient = \"h\")\nax0_sns.set_ylabel(\"Feature name\", fontsize = 4.5)\nax0_sns.set_xlabel(\"Feature importance (7 folds, ratio of total)\", fontsize = 4.5)\nax0_sns.tick_params(labelsize = 3.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:25:59.219143Z","iopub.status.idle":"2022-09-17T12:25:59.220122Z","shell.execute_reply.started":"2022-09-17T12:25:59.219886Z","shell.execute_reply":"2022-09-17T12:25:59.219918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict results (mean win bid of each k-model)","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({'deviceId':test_feats['deviceId'], 'floor':test_feats['bidFloorPrice'], 'winBid':0})\n\nsubmission['winBid'] = np.mean(np.column_stack(predictions), axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:25:59.221150Z","iopub.status.idle":"2022-09-17T12:25:59.221919Z","shell.execute_reply.started":"2022-09-17T12:25:59.221661Z","shell.execute_reply":"2022-09-17T12:25:59.221692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clip results from 0.01 and create submission file","metadata":{}},{"cell_type":"code","source":"submission['winBid'] = np.clip(submission['winBid'], submission['floor'], 3800)\n\nsubmission = submission[['deviceId', 'winBid']]\nsubmission.to_csv(\"submission.csv\", index = None)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-09-17T12:25:59.223520Z","iopub.status.idle":"2022-09-17T12:25:59.223932Z","shell.execute_reply.started":"2022-09-17T12:25:59.223722Z","shell.execute_reply":"2022-09-17T12:25:59.223740Z"},"trusted":true},"execution_count":null,"outputs":[]}]}