{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random, os, gc\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_columns', None)\n\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import PolynomialFeatures\n\nimport lightgbm as lgb\nfrom lightgbm import early_stopping, log_evaluation\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set(rc={'figure.figsize':(16,12)})\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\ndef gc_clear():\n    for i in range(5):\n        gc.collect()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"../input/digital-turbine-auction-bid-price-prediction/\"\ntrain = pd.read_csv(data_path + \"train_data.csv\").sort_values(\"eventTimestamp\")\ntest = pd.read_csv(data_path + \"test_data.csv\")\nss = pd.read_csv(data_path + \"sample_submission.csv\")\n\ntrain['bidrank'] = train.groupby('deviceId')['eventTimestamp'].rank(\n    method='first',ascending=False\n).astype(int)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cat_encode(train, test, colname, topN=-1):\n    '''\n    Encodes topN categories from train_set \n    of \"colname\" column\n    and merge all other categories into one (if topN!=-1).\n    '''\n    top_df = pd.DataFrame(\n        train[train.bidrank==1][colname].value_counts()\n    ).reset_index().reset_index()\n    if topN!=-1:\n        top_df = top_df[:topN]\n    else:\n        topN = len(top_df)\n    col_dict = dict(zip(top_df['index'], top_df['level_0']))\n    train[colname] = train[colname].map(col_dict).fillna(topN).astype(int)\n    test[colname] = test[colname].map(col_dict).fillna(topN).astype(int)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ENCODING_LIST = [\n    ('unitDisplayType',-1),\n    ('brandName',-1),\n    ('bundleId',-1),\n    ('countryCode',6),\n    ('osAndVersion',-1),\n    ('connectionType',-1),\n    ('c1',-1),\n    ('c3',-1),\n    ('size',-1),\n]\n\nfor col, topN in ENCODING_LIST:\n    print(f\"Encoding {col} column ...\")\n    cat_encode(train, test, col, topN)\nprint(\"Done.\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dictionary with grouping levels and aggregations for feature engineering\nGROUP_DICTS = [\n    {\n        'group_level': ['deviceId'],\n        'prefix': \"device_\",\n        'aggregations': {\n            'bidFloorPrice':['median'],\n            'sentPrice':['mean','median','last'],\n            'winBid':['std','var','min','max','mean','median','last','count', lambda x: x.quantile(0.75), lambda x: x.quantile(0.25)],\n            'eventTimestamp':['last'],\n            'timediff':['mean','median'],\n        }\n    },\n    {\n        'group_level': ['deviceId','unitDisplayType'],\n        'prefix': 'device_display_',\n        'aggregations': {\n            'sentPrice':['min','max','last','mean','median'],\n            'winBid':['std','var','min','max','mean','median','last','count', lambda x: x.quantile(0.75), lambda x: x.quantile(0.25)],\n            'eventTimestamp':['last'],\n            'timediffU':['mean','median'],\n        }\n    },\n    {\n        'group_level': ['countryCode','bundleId'],\n        'prefix': 'country_bundle_',\n        'aggregations': {\n            'bidFloorPrice':['mean'],\n            'winBid':['std','min','max','count'],        \n        }\n    },\n    {\n        'group_level': ['countryCode'],\n        'prefix': 'country_',\n        'aggregations': {\n            'sentPrice':['mean'],\n            'winBid':['std','var','max'],        \n        }\n    },\n    {\n        'group_level': ['countryCode','connectionType'],\n        'prefix': 'country_connection_',\n        'aggregations': {\n            'sentPrice':['max'],\n            'winBid':['max'],        \n        }\n    }\n]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# main function for feature engineering\ndef feat_eng(df_history, df_current):\n    fe = df_current[[\n        'eventTimestamp','deviceId','unitDisplayType','countryCode','connectionType',\n        'bidFloorPrice','sentPrice','c1','c3','bundleId',\n        'brandName','osAndVersion','size'\n    ]]\n\n    df_history['timediff'] = df_history.groupby(['deviceId'])['eventTimestamp'].diff()\n    df_history['timediffU'] = df_history.groupby(['deviceId','unitDisplayType'])['eventTimestamp'].diff()\n    for group_dict in GROUP_DICTS:\n        gr_df = df_history.groupby(\n            group_dict['group_level']\n        ).agg(\n            group_dict['aggregations']\n        )\n        gr_df.columns = ['_'.join(x) for x in gr_df.columns]\n        gr_df.columns = [group_dict['prefix']+x for x in gr_df.columns]\n        gr_df.reset_index(inplace=True)\n\n        fe = fe.merge(gr_df, on=group_dict['group_level'], how='left')\n        \n    df_history['display_bidrank'] = df_history.groupby(\n        ['deviceId','unitDisplayType']\n    )['eventTimestamp'].rank(method='first',ascending=False).astype(int)\n    dfs = df_history[\n        df_history['display_bidrank']==2\n    ][['deviceId','unitDisplayType','winBid']]\n    dfs.columns = ['deviceId','unitDisplayType','device_display_winBid_last2']\n    fe = fe.merge(dfs, on=['deviceId','unitDisplayType'], how='left')\n    del df_history['display_bidrank']\n    \n    fe['estimation_1'] = (fe['device_winBid_last']/fe['device_sentPrice_last']) * fe['sentPrice']\n    fe['estimation_2'] = (fe['device_winBid_mean']/fe['device_sentPrice_mean']) * fe['sentPrice']\n    fe['estimation_3'] = (fe['device_winBid_median']/fe['device_sentPrice_median']) * fe['sentPrice']\n    fe['estimation_4'] = (fe['device_winBid_last']-fe['device_sentPrice_last']) + fe['sentPrice']\n    fe['estimation_5'] = (fe['device_winBid_mean']-fe['device_sentPrice_mean']) + fe['sentPrice']\n    fe['estimation_6'] = (fe['device_winBid_median']-fe['device_sentPrice_median']) + fe['sentPrice']\n    fe['estimation_4b'] = (fe['device_display_winBid_last']-fe['device_display_sentPrice_last']) + fe['sentPrice']\n    fe['estimation_5b'] = (fe['device_display_winBid_mean']-fe['device_display_sentPrice_mean']) + fe['sentPrice']\n    fe['estimation_6b'] = (fe['device_display_winBid_median']-fe['device_display_sentPrice_median']) + fe['sentPrice']\n    \n    fe['estimation_min'] = fe[['estimation_1','estimation_2','estimation_3']].min(axis=1)\n    fe['estimation_max'] = fe[['estimation_1','estimation_2','estimation_3']].max(axis=1)\n    fe['estimation_mean'] = fe[['estimation_1','estimation_2','estimation_3']].mean(axis=1)\n    \n    fe['device_eventTimestamp_vs_last'] = fe['eventTimestamp'] - fe['device_eventTimestamp_last']\n    fe['device_display_eventTimestamp_vs_last'] = fe['eventTimestamp'] - fe['device_display_eventTimestamp_last']\n    \n    # drop some useless features according to Permutation Feature Importance\n    fe.drop(\n        columns = [\n            'device_sentPrice_last','device_sentPrice_mean','device_sentPrice_median',\n            'device_display_sentPrice_mean','device_display_sentPrice_median',\n            'eventTimestamp','device_eventTimestamp_last','device_display_eventTimestamp_last',\n        ],\n        inplace=True\n    )\n        \n    return fe","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fes = []\nfor i in range(1,11):\n    trainy = train[train.bidrank>i].copy()\n    val = train[train.bidrank==i].copy()\n    val = trainy[['deviceId']].drop_duplicates().reset_index(drop=True).merge(val, on=['deviceId'], how='left')\n    gc_clear()\n    \n    fe = feat_eng(trainy, val)\n    fe = fe.merge(\n        val[['deviceId','winBid']], \n        on=['deviceId'], \n        how='left'\n    )\n    fes.append(fe)\n    del fe\n    gc_clear()\n    print(f\"FE for i={i}: done\")\n\nfe = pd.concat(fes,ignore_index=True)\n\ntest_fe = feat_eng(train, test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats = [col for col in fe.columns if col not in [\n    'deviceId','winBid','bundleId','c3','device_display_winBid_last2','connectionType'\n]]\ncatfeats = ['unitDisplayType','countryCode','c1','brandName','osAndVersion','size']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_lgbm(train, test, conf):\n    seed_everything(conf.params['seed'])\n    \n    features = conf.features\n    cat_features = conf.cat_features\n    num_features = len(features)\n\n    if conf.predict_test:\n        test_predictions = np.zeros(len(test))\n    oof_predictions = np.zeros(len(train))\n    feature_importance_df = pd.DataFrame()\n    \n    kfold = KFold(n_splits = conf.n_folds, shuffle = True, random_state = conf.params['seed'])\n\n    for fold, (trn_ind, val_ind) in enumerate(kfold.split(train, train[conf.target])):\n        print('\\n'+'-'*50 + f'\\nTraining fold {fold} with {num_features} features...\\n')\n        x_train, x_val = train[features].iloc[trn_ind], train[features].iloc[val_ind]\n        y_train, y_val = train[conf.target].iloc[trn_ind], train[conf.target].iloc[val_ind]\n        lgb_train = lgb.Dataset(x_train, y_train, categorical_feature = cat_features)\n        lgb_valid = lgb.Dataset(x_val, y_val, categorical_feature = cat_features)\n        model = lgb.train(\n            params = conf.params,\n            train_set = lgb_train,\n            num_boost_round = conf.trees,\n            valid_sets = [lgb_train, lgb_valid],\n            categorical_feature = cat_features,\n            callbacks=[\n                log_evaluation(conf.verbose),\n            ] if conf.params[\"boosting\"]==\"dart\" else [\n                log_evaluation(conf.verbose),\n                early_stopping(conf.early,verbose=True)                \n            ]\n        )\n        \n        fold_importance_df = pd.DataFrame()\n        fold_importance_df[\"feature\"] = model.feature_name()\n        fold_importance_df[\"importance\"] = model.feature_importance(importance_type='split')\n        fold_importance_df[\"importance2\"] = model.feature_importance(importance_type='gain')\n        fold_importance_df[\"fold\"] = fold\n        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n        \n        val_pred = model.predict(x_val)\n        oof_predictions[val_ind] = val_pred\n\n        if conf.predict_test:\n            test_predictions += model.predict(test[features]) / conf.n_folds\n        \n        score = mean_squared_error(y_val, val_pred, squared=False)\n        print(f'\\nFold {fold} CV score is {score:.2f}')\n        \n        del x_train, x_val, y_train, y_val, lgb_train, lgb_valid\n        gc_clear()\n        \n    oof = pd.DataFrame({'deviceId': train['deviceId'], 'winBid': train['winBid'], 'sentPrice': train['sentPrice'], 'pred': oof_predictions})\n    oof['pred'] = oof[['pred', 'sentPrice']].max(axis=1)\n    \n    score = mean_squared_error(train[conf.target], oof_predictions, squared=False)\n    score2 = mean_squared_error(oof['winBid'], oof['pred'], squared=False)\n    print(f'\\nOut of folds CV score is {score:.2f}; corrected score is  {score2:.2f}')\n    \n    if conf.predict_test:\n        sub = pd.DataFrame({'deviceId': test['deviceId'], 'winBid': test_predictions})\n        #sub.to_csv(f\"submission_{score:.2f}_{conf.params['boosting']}_{conf.n_folds}folds_seed{conf.params['seed']}.csv\", index = False)\n    display_importances(feature_importance_df)\n    return oof, sub\n    \n\ndef display_importances(feature_importance_df_):\n    tops = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:80].index\n    tops2 = feature_importance_df_[[\"feature\", \"importance2\"]].groupby(\"feature\").mean().sort_values(by=\"importance2\", ascending=False)[:80].index\n    \n    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(tops)]\n    best_features2 = feature_importance_df_.loc[feature_importance_df_.feature.isin(tops2)]\n    \n    plt.figure(figsize=(8, 15))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n    plt.title('LightGBM Features Split (avg over folds)')\n    plt.tight_layout()\n    plt.show()\n    \n    plt.figure(figsize=(8, 15))\n    sns.barplot(x=\"importance2\", y=\"feature\", data=best_features2.sort_values(by=\"importance2\", ascending=False))\n    plt.title('LightGBM Features Gain (avg over folds)')\n    plt.tight_layout()\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    n_folds = 3\n    trees = 4500\n    early = 5000\n    verbose = 100\n    params = {\n        'objective': 'rmse',\n        'metric': 'rmse',\n        'boosting': 'dart',\n        'seed': 42,\n        'num_leaves': 16,\n        'learning_rate': 0.01,\n        'feature_fraction': 0.40,\n        'n_jobs': -1,\n        'lambda_l2': 2,\n        'min_data_in_leaf': 40,\n    }\n    target = 'winBid'\n    features = feats\n    cat_features = catfeats\n    predict_test = True","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"C1 = CFG()\n\noofs_mult, subs_mult = [], []\n\nfor unitDisplayType in [0,1,2]:\n    print(f\"Train model for unitDisplayType={unitDisplayType}\")\n    \n    fe_ = fe[fe.unitDisplayType==unitDisplayType].copy()\n    test_fe_ = test_fe[test_fe.unitDisplayType==unitDisplayType].copy()\n\n    oofs, subs = [], []\n    for seed in [42,4,8]:\n        C1.params['seed'] = seed\n        oof, sub = run_lgbm(fe_, test_fe_, C1)\n        oofs.append(oof)\n        subs.append(sub)\n        \n    oofs_mult.append(oofs)\n    subs_mult.append(subs)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_lgb = []\nsub_lgb = []\n\nfor i in range(len(oofs_mult)):\n    print(f\"i={i}\")\n    \n    oofs = oofs_mult[i]\n    subs = subs_mult[i]\n    \n    oof = oofs[0].copy()\n    oof['pred'] = 0\n    for o in oofs:\n        oof['pred'] += o['pred'] / len(oofs)\n\n    sub = subs[0].copy()\n    sub['winBid'] = 0\n    for s in subs:\n        sub['winBid'] += s['winBid'] / len(subs)\n\n    cv = mean_squared_error(oof['winBid'],oof['pred'],squared=False)\n    print(cv)\n    \n    oof_lgb.append(oof)\n    sub_lgb.append(sub)\n    \noof = pd.concat(oof_lgb,ignore_index=True)\nsub = pd.concat(sub_lgb,ignore_index=True)\n\ncv = mean_squared_error(oof['winBid'],oof['pred'],squared=False)\nprint(cv)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_lgbm = ss[['deviceId']].merge(sub[['deviceId','winBid']],on=['deviceId'],how='left')\nsub_lgbm = sub_lgbm.merge(test[['deviceId','sentPrice']], on=['deviceId'], how='left')\nsub_lgbm['winBid'] = sub_lgbm[['winBid','sentPrice']].max(axis=1)\nsub_lgbm[['deviceId','winBid']].to_csv(f'submission_lgbm_{cv:.4f}.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]}]}