{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-28T10:17:11.461273Z","iopub.execute_input":"2023-09-28T10:17:11.461603Z","iopub.status.idle":"2023-09-28T10:17:11.475169Z","shell.execute_reply.started":"2023-09-28T10:17:11.461579Z","shell.execute_reply":"2023-09-28T10:17:11.473691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Importing libraries**","metadata":{}},{"cell_type":"code","source":"import matplotlib\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import make_axes_locatable\nimport random\nimport datetime\nfrom itertools import accumulate\n\nfrom sklearn.feature_selection import mutual_info_regression\nfrom sklearn.preprocessing import OrdinalEncoder,OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nimport lightgbm as lgb\nfrom lightgbm import early_stopping\nfrom lightgbm import log_evaluation","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:17:11.477292Z","iopub.execute_input":"2023-09-28T10:17:11.477624Z","iopub.status.idle":"2023-09-28T10:17:12.547138Z","shell.execute_reply.started":"2023-09-28T10:17:11.477592Z","shell.execute_reply":"2023-09-28T10:17:12.545317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Installing survival analysis library**","metadata":{}},{"cell_type":"code","source":"# !pip install scikit-survival\n# from sksurv.ensemble import ComponentwiseGradientBoostingSurvivalAnalysis as cgbsa\n# from sksurv.ensemble import GradientBoostingSurvivalAnalysis as gbsa\n# from sksurv.linear_model import CoxPHSurvivalAnalysis, CoxnetSurvivalAnalysis\n# from sksurv.preprocessing import OneHotEncoder\n# from sksurv.column        import standardize\n# from sksurv.metrics import concordance_index_censored\n# from sksurv.compare import compare_survival","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:17:12.548951Z","iopub.execute_input":"2023-09-28T10:17:12.549415Z","iopub.status.idle":"2023-09-28T10:17:12.555141Z","shell.execute_reply.started":"2023-09-28T10:17:12.549377Z","shell.execute_reply":"2023-09-28T10:17:12.553551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 0. Some functions for data handling\n**This section can be ignored. Come back to it if need to clarify something.**","metadata":{}},{"cell_type":"markdown","source":"**First functions realted with encoders.**","metadata":{}},{"cell_type":"code","source":"def get_df_encoders(\n                    working_data : \"DataFrame\",  # Data to be transformed\n                    feature_classes : \"dict\"\n                    ):\n    \"\"\" Generating the encoders to transform\n        the data into its training format.\n        The marketWgts dictionary required as input for\n        transformation of testing/validation sets.\n        During train set transformation the marketWgts dict\n        is generated.\n\n        The maketWgts dictionary correponds to:\n            dic[feature_value] = (#_feature_value)/total_winbids_df\n\n        Returns (encoding_objects): transformed data an a tuple\n                                    of objects used to transform data.\n    \"\"\"\n    marketWgts = {}\n    cat_encoder = OneHotEncoder()\n\n    # When : Incorporating hour and day of week on which auction occurred.\n    working_data[\"hour\"] = working_data.eventTimestamp.dt.hour\n    working_data[\"wday\"] = working_data.eventTimestamp.dt.dayofweek\n\n    # Merging : The appversion only make sense in relation with the app.\n    working_data[\"appIdVer\"] = working_data[\"bundleId\"] + working_data[\"appVersion\"]\n\n    # Simplyfing : Considering beta versions as unstable.\n    working_data[\"MPVstable\"] = working_data[\"mediationProviderVersion\"].str.contains('beta')\n\n    # Defining column types: grouping the different types of columns\n    col_categorical = feature_classes[\"categorical\"]\n    col_cate_market = feature_classes[\"categorical_market\"]\n    col_ready = feature_classes[\"ready\"]\n    col_monitor = feature_classes[\"monitor\"]\n\n    # Defining grouping the features\n    cols_feat = col_categorical + col_cate_market + col_ready\n\n    ###################################\n    ### Transforming the columns ######\n\n    # Transforming the cate_market columns\n    winValue = working_data[working_data.winBid<=50].winBid.sum()\n    marketWgts = {}\n\n    for colname in col_cate_market:\n        raw_val_dict = working_data[colname].value_counts().to_dict() # Counts feat. value\n        col_values = list(raw_val_dict.keys()) # feat values\n        n_keys = len(col_values)\n        for key_i in range(n_keys):\n            value_in_market = raw_val_dict[col_values[key_i]]/winValue  # feat values/sum_of_winBids\n            raw_val_dict[col_values[key_i]] = np.round(value_in_market,decimals=6)\n        marketWgts[colname] = raw_val_dict # storing the weighted histogram\n\n    # One_hot_encoding of categorical columns\n\n    for colname in col_categorical:\n        working_data[colname] = working_data[colname].astype(\"category\")\n    cat_encoder.fit(working_data[col_categorical])\n\n    return (marketWgts,cat_encoder)\n\ndef apply_encoders(working_data,feature_classes):\n    working_data[\"hour\"] = working_data.eventTimestamp.dt.hour\n    working_data[\"wday\"] = working_data.eventTimestamp.dt.dayofweek\n\n    # Merging : The appversion only make sense in relation with the app.\n    working_data[\"appIdVer\"] = working_data[\"bundleId\"] + working_data[\"appVersion\"]\n\n    # Simplyfing : Considering beta versions as unstable.\n    working_data[\"MPVstable\"] = working_data[\"mediationProviderVersion\"].str.contains('beta')\n\n    # Defining column types: grouping the different types of columns\n    col_categorical = feature_classes[\"categorical\"]\n    col_cate_market = feature_classes[\"categorical_market\"]\n    col_ready = feature_classes[\"ready\"]\n    col_monitor = feature_classes[\"monitor\"]\n\n    # Defining grouping the features\n\n    cols_feat = col_categorical + col_cate_market + col_ready\n    for colname in col_categorical:\n        working_data[colname] = working_data[colname].astype(\"category\")\n    return working_data[cols_feat+col_monitor]\n\ndef transform_df(\n                    working_data : \"DataFrame\",    # Data to be transformed\n                    encoders : \"tuple\" ,         # (Market Value weigths, categorical encoder )\n                    feature_classes: \"dict\", # features names dictionary\n                    ):\n    \"\"\" Transforming the data into its training format.\n        The marketWgts dictionary and categorical encoder, are\n        required as input for transformation. Those objects\n        come from get_df_encoders.\n\n        Returns (DataFrame): transformed data an a tuple\n                             of objects used to transform data.\n    \"\"\"\n    marketWgts = encoders[0]\n    cat_encoder = encoders[1]\n\n    # Defining column types: grouping the different types of columns\n    col_categorical = feature_classes[\"categorical\"]\n    col_cate_market = feature_classes[\"categorical_market\"]\n    col_ready = feature_classes[\"ready\"]\n    col_monitor = feature_classes[\"monitor\"]\n    cols_feat = col_categorical + col_cate_market + col_ready\n    \n    # Defining grouping the features and columns to be dropped\n\n    working_data = apply_encoders(working_data.copy(),feature_classes)\n    \n    cols_drop = [ i for i in working_data.columns\n                if not i in cols_feat+col_monitor ]\n\n    ###################################\n    ### Transforming the columns ######\n\n    # Transforming the cate_market columns\n    for colname in col_cate_market:\n        col_market_vals = marketWgts[colname]    \n        temp = [ col_market_vals.get(col_val,0) for col_val in working_data[colname].values ]\n        working_data[colname] = temp\n    ### Dropping columns and rearranging\n    working_data = working_data.drop(columns=cols_drop)\n\n    working_feats = list(set(cols_feat+col_monitor))\n    working_data = working_data[working_feats]\n\n    # One_hot_encoding of categorical columns\n    if len(col_categorical) :\n        cat_cols = pd.DataFrame.sparse.from_spmatrix(\n                            cat_encoder.transform(\\\n                            pd.DataFrame(working_data[col_categorical])),\n                            index=working_data.index,\n                            columns=cat_encoder.categories_[0])\n        return pd.concat([working_data.drop(columns=col_categorical),cat_cols],axis=1)\n    else:\n        return working_data\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:17:12.557962Z","iopub.execute_input":"2023-09-28T10:17:12.558381Z","iopub.status.idle":"2023-09-28T10:17:12.579448Z","shell.execute_reply.started":"2023-09-28T10:17:12.558336Z","shell.execute_reply":"2023-09-28T10:17:12.577871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Functions related with plotting a visualization**","metadata":{}},{"cell_type":"code","source":"def plot_pie_feat(df,feat,pltobj=None):\n    if pltobj == None:\n        fig = plt.figure(figsize=(4,4))\n        pltobj = fig.add_subplot(111)\n    dictfeat = df[feat].value_counts().to_dict()\n    pltobj.pie(dictfeat.values(),labels=dictfeat.keys())\n\ndef smoothcurve(data,bins,degree=5):\n    \"\"\"Smooting the curve only for visual inspection.\"\"\"\n    bins = np.array(bins)\n    mptn = (bins[1]-bins[0])*0.5\n    bins = bins[:-1]+mptn\n    weights = np.exp(-(np.arange(degree)-degree//2)**2 / (2*(degree/3)**2))\n    weights /= np.sum(weights)\n    smoothed = np.convolve(data, weights, mode='same')\n    return smoothed,bins\n\ndef plot_scatter_subplot(pltobj,df,ft1,ft2,\n                         ylog=False,xlog=False,\n                         xlims=None,\n                         ylims=None,\n                         xlabel=\"\",\n                         ylabel=\"\",\n                        ):\n    xdata = df[ft1]\n    ydata = df[ft2]\n    pltobj.scatter(xdata,ydata,alpha=0.1)\n    if xlog:\n        plt.xscale(\"log\")\n    if ylog:\n        plt.yscale(\"log\")\n    if xlims != None:\n        pltobj.set_xlim(xlims)\n    if ylims != None:\n        pltobj.set_ylim(ylims)\n    if xlabel != \"\":\n        pltobj.set_xlabel(xlabel)\n    if ylabel != \"\":\n        pltobj.set_ylabel(ylabel)\n        \ndef plot_hist(pltobj,df,ft1,\n            color='k',\n            nbins=100,\n            range=(0,100),\n            xlabel=\"\",\n            ylabel=\"\",\n            density=True ):\n    if isinstance(nbins, list):\n        n , bins = np.histogram(df[ft1],bins=nbins,density=density)\n    else:\n        n , bins = np.histogram(df[ft1],bins=nbins,range=range,density=density)\n    n , bins = smoothcurve(n,bins)\n    pltobj.plot(bins,n,ls='-',lw=2,color=color)\n    if xlabel != \"\":\n        pltobj.set_xlabel(xlabel)\n    if ylabel != \"\":\n        pltobj.set_ylabel(ylabel)\n\ndef plot_scatter_subplot_hist(pltobj,df,ft1,ft2,\n                        xhist_range,\n                        xhist_label=\"\",\n                        xtitle=\"\",\n                        ylog=False,xlog=False,\n                        xlims=None,\n                        ylims=None,\n                        xlabel=\"\",\n                        ylabel=\"\",\n                        ):\n    xdata = df[ft1]\n    ydata = df[ft2]\n    pltobj.scatter(xdata,ydata,alpha=0.1)\n    if xlog:\n        plt.xscale(\"log\")\n    if ylog:\n        plt.yscale(\"log\")\n    if xlims != None:\n        pltobj.set_xlim(xlims)\n    if ylims != None:\n        pltobj.set_ylim(ylims)\n    if xlabel != \"\":\n        pltobj.set_xlabel(xlabel)\n    if ylabel != \"\":\n        pltobj.set_ylabel(ylabel)\n    plt.grid()\n\n    divider = make_axes_locatable(pltobj)\n    ax_histx = divider.append_axes(\"top\", 0.65, pad=0.1, sharex=pltobj)\n    ax_histx.xaxis.set_tick_params(labelbottom=False)\n    \n    plot_hist(ax_histx,df,ft1,\n        color='k',\n        nbins=xhist_range,\n        xlabel=\"\",\n        ylabel=xhist_label,\n        density=False)\n    \n    if xtitle != \"\":\n        ax_histx.set_title(xtitle)\n\n#     ax_histy = divider.append_axes(\"right\", 0.65, pad=0.1, sharey=pltobj)\n#     ax_histy.yaxis.set_tick_params(labelleft=False)\n#     plot_hist(ax_histy,df,ft2,\n#     color='k',\n#     nbins=yhist_range,\n#     xlabel=\"\",\n#     ylabel=\"\")","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:17:12.581270Z","iopub.execute_input":"2023-09-28T10:17:12.581740Z","iopub.status.idle":"2023-09-28T10:17:12.601464Z","shell.execute_reply.started":"2023-09-28T10:17:12.581703Z","shell.execute_reply":"2023-09-28T10:17:12.599983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Additional helper functions**","metadata":{}},{"cell_type":"code","source":"def get_col_dicts(working_data,columns_to_reduce,n_samples,cutoff=0.9):\n    diccol = {}\n    for col in columns_to_reduce:\n        dictest = relevant_instances(working_data,col,cutoff)\n        dicndx = {}\n        for key,val in dictest.items():\n            expected = int(val*n_samples)\n            filtered = working_data[working_data[col]==key].index\n            selected = np.random.choice(filtered,expected,replace=False)\n            dicndx[key] = list(selected)\n        diccol[col] = dicndx\n    return diccol\n\ndef ndx_to_work(diccol):\n    seen_ndx = set(ndx for dic in diccol for key in diccol[dic].values() for ndx in key)\n    return seen_ndx\n\ndef eval_model(model,data2eval,corr_type='log'):\n    prediction = model.predict(data2eval,num_iteration=model.best_iteration)\n    if corr_type == 'log':\n        prediction = np.exp(prediction)-1\n    prediction = np.where(prediction>=0.01,np.round(prediction,2),0.001)\n    return prediction\n\ndef relevant_instances(df, col, cutoff=0.90):\n    distrib = df[col].value_counts(normalize=True).to_dict() \n    relevant_values = {}\n\n    for value, count in zip(distrib.keys(), accumulate(distrib.values())):\n        relevant_values[value] = distrib[value]\n        if count >= cutoff:\n            break\n\n    return relevant_values","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:17:12.603136Z","iopub.execute_input":"2023-09-28T10:17:12.603541Z","iopub.status.idle":"2023-09-28T10:17:12.619899Z","shell.execute_reply.started":"2023-09-28T10:17:12.603512Z","shell.execute_reply":"2023-09-28T10:17:12.619048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Exploration\n**Identifying the features and characteristics.**","metadata":{}},{"cell_type":"code","source":"datadir=\"/kaggle/input/digital-turbine-auction-bid-price-prediction\"\noridf = pd.read_csv(\"%s/train_data.csv\"%datadir)\noridf['eventTimestamp'] = pd.to_datetime(oridf['eventTimestamp'],unit='ms')\noridf = oridf.sort_values(by=['eventTimestamp'])\noridf = oridf.dropna().reset_index(drop=True)\noridf.set_index('eventTimestamp', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:17:12.620911Z","iopub.execute_input":"2023-09-28T10:17:12.621178Z","iopub.status.idle":"2023-09-28T10:18:03.840601Z","shell.execute_reply.started":"2023-09-28T10:17:12.621157Z","shell.execute_reply":"2023-09-28T10:18:03.839008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***First exploring distribution within some features.***","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,5))\nax = fig.add_subplot(121)\nplot_pie_feat(oridf,\"unitDisplayType\",ax)\nax.set_title(\"unitDisplayType\")\nax = fig.add_subplot(122)\nplot_pie_feat(oridf,\"bidFloorPrice\",ax)\nax.set_title(\"bidFloorPrice\")","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:18:03.842246Z","iopub.execute_input":"2023-09-28T10:18:03.842854Z","iopub.status.idle":"2023-09-28T10:18:04.535767Z","shell.execute_reply.started":"2023-09-28T10:18:03.842820Z","shell.execute_reply":"2023-09-28T10:18:04.534599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gen_data = oridf.iloc[::100,:] # subsampling\nmaxval = oridf.winBid.max()\nhist_nbins = [0.001,0.005]+\\\n            list(np.logspace(0,np.log10(100),20)*.01)+\\\n            list(np.logspace(0,np.log10(1000),50))[1:]\n\nfig = plt.figure(figsize=(14,5))\n\nax = fig.add_subplot(141)\ndata = gen_data[(gen_data[\"unitDisplayType\"]=='banner')]\nplot_scatter_subplot_hist(ax,data,\"winBid\",\"sentPrice\",\n                    xhist_range=hist_nbins,\n                    xhist_label=\"Counts\",\n                    xtitle=\"banner\",\n                    xlog=True,ylog=True,\n                    xlims=[0.01,maxval],ylims=[0.01,maxval],\n                    xlabel=\"Win Bid  [ $ CPM ]\",\n                    ylabel=\"Sent Price  [ $ CPM ]\")\n# \n\nax = fig.add_subplot(142)\ndata = gen_data[(gen_data[\"unitDisplayType\"]=='rewarded')]\nplot_scatter_subplot_hist(ax,data,\"winBid\",\"sentPrice\",\n                    xhist_range=hist_nbins,\n                    xtitle=\"rewarded\",\n                    xlog=True,ylog=True,\n                    xlims=[0.01,maxval],ylims=[0.01,maxval],\n                    xlabel=\"Win Bid  [ $ CPM ]\")\n\nax = fig.add_subplot(143)\ndata = gen_data[(gen_data[\"unitDisplayType\"]=='interstitial')]\nplot_scatter_subplot_hist(ax,data,\"winBid\",\"sentPrice\",\n                    xhist_range=hist_nbins,\n                    xtitle=\"interstitial\",\n                    xlog=True,ylog=True,\n                    xlims=[0.01,maxval],ylims=[0.01,maxval],\n                    xlabel=\"Win Bid  [ $ CPM ]\")\n\nax = fig.add_subplot(144)\n\ndata = gen_data[(gen_data.unitDisplayType=='banner')].copy()\ndata[\"winBid\"] *= 10\nplot_hist(ax,data,\"winBid\",color='r',nbins=100)\ndel data\n\ndata = gen_data[(gen_data.unitDisplayType=='rewarded')]\nplot_hist(ax,data,\"winBid\",color='k',nbins=100)\n\ndata = gen_data[(gen_data.unitDisplayType=='interstitial')]\nplot_hist(ax,data,\"winBid\",color='b',nbins=100,\n          xlabel=\"Win Bid  [ $ CPM ]\")\n\nax.legend(['banner X10','rewarded','interstitial'])\nax.set_title('Distribution')","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:18:04.540073Z","iopub.execute_input":"2023-09-28T10:18:04.540814Z","iopub.status.idle":"2023-09-28T10:18:07.058268Z","shell.execute_reply.started":"2023-09-28T10:18:04.540783Z","shell.execute_reply":"2023-09-28T10:18:07.056719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['unitDisplayType', 'brandName',\n        'bundleId', 'appVersion',\n        'correctModelName', 'countryCode',\n         'osAndVersion', \n        'connectionType', \n        'c1', 'c2', 'c3', 'c4',\n        'size', 'mediationProviderVersion',\"bidFloorPrice\"]\n\noencoder = OrdinalEncoder()\noencoder.fit(oridf[cols])","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:18:07.059810Z","iopub.execute_input":"2023-09-28T10:18:07.060483Z","iopub.status.idle":"2023-09-28T10:18:14.045406Z","shell.execute_reply.started":"2023-09-28T10:18:07.060445Z","shell.execute_reply":"2023-09-28T10:18:14.044099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ndxs = np.random.choice(range(oridf.shape[0]),100000,replace=False)\nsubsample = oridf.iloc[ndxs,:]\nfet = oencoder.transform(subsample[cols])\ntar = subsample.winBid\nnfeat = range(fet.shape[1])\nadtype = list(oencoder.categories_[0])\n\nmutinfo = {}\n\nvals = [ mutual_info_regression(fet[:,k].reshape(-1,1),tar) for k in nfeat ]\nmutinfo[\"all\"] = vals\nfor i,udt in enumerate(adtype):\n    mask = (fet[:,0]==i)\n    fff = np.array([ n for j,n in enumerate(fet) if mask[j] ])\n    ttt = [ n for j,n in enumerate(tar) if mask[j] ]\n    vals = [ mutual_info_regression(fff[:,k].reshape(-1,1),ttt) for k in nfeat ]\n    mutinfo[udt] = vals\ndel fff,ttt,vals,mask","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:18:14.047620Z","iopub.execute_input":"2023-09-28T10:18:14.047986Z","iopub.status.idle":"2023-09-28T10:18:35.594088Z","shell.execute_reply.started":"2023-09-28T10:18:14.047962Z","shell.execute_reply":"2023-09-28T10:18:35.592130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = np.arange(len(adtype)+1)  # the label locations\nwidth = 0.05  # the width of the bars\nmultiplier = 0\n\nfig = plt.figure(figsize=[9,4])\nax = fig.add_subplot(111)\n\nfor i, k in enumerate(cols):\n    offset = width * multiplier\n    dat = [ mutinfo[c][i][0] for c in [\"all\"]+adtype ] \n    rects = ax.bar(x + offset, dat, width, label=k)\n    #ax.bar_label(rects, padding=3)\n    multiplier += 1\nplt.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\nax.set_xticks(x + width*6, [\"all\"]+adtype)\nax.set_ylabel('Mutual Information')\nax.set_title('Mut. Inf. against WinBid ')","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:18:35.595796Z","iopub.execute_input":"2023-09-28T10:18:35.596958Z","iopub.status.idle":"2023-09-28T10:18:36.019145Z","shell.execute_reply.started":"2023-09-28T10:18:35.596905Z","shell.execute_reply":"2023-09-28T10:18:36.017678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['unitDisplayType', 'brandName',\n        'bundleId', 'appVersion',\n        'correctModelName', 'countryCode',\n         'osAndVersion', \n        'c1', 'c4', \n        'size', 'mediationProviderVersion']\n\nhist_dicts_b = {}\nfor colname in cols:\n        hist_dicts_b[colname] = oridf[(oridf.unitDisplayType=='banner')][colname].value_counts().to_dict()\nhist_dicts_i = {}\nfor colname in cols:\n        hist_dicts_i[colname] = oridf[(oridf.unitDisplayType=='interstitial')][colname].value_counts().to_dict()\nhist_dicts_r = {}\nfor colname in cols:\n        hist_dicts_r[colname] = oridf[(oridf.unitDisplayType=='rewarded')][colname].value_counts().to_dict()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:18:36.020893Z","iopub.execute_input":"2023-09-28T10:18:36.021236Z","iopub.status.idle":"2023-09-28T10:19:17.223142Z","shell.execute_reply.started":"2023-09-28T10:18:36.021209Z","shell.execute_reply":"2023-09-28T10:19:17.221923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adtype = ['banner','interstitial','rewared']\nfeatnames = cols\nfeat_vals = { f : (len(hist_dicts_b[f].values()),\n                       len(hist_dicts_i[f].values()),\n                       len(hist_dicts_r[f].values())) for f in featnames }\n\nx = np.arange(len(adtype))  # the label locations\nwidth = 0.05  # the width of the bars\nmultiplier = 0\n\nfig = plt.figure(figsize=[9,4])\nax = fig.add_subplot(111)\n\nfor attribute, measurement in feat_vals.items():\n    offset = width * multiplier\n    rects = ax.bar(x + offset, measurement, width, label=attribute)\n    ax.bar_label(rects, padding=3)\n    multiplier += 1\n\n# Add some text for labels, title and custom x-axis tick labels, etc.\nax.set_ylabel('# Values in feature')\n#ax.set_xlabel('Unit Display Type')\nax.set_title('Distribution of features. values over Ad Type')\nax.set_xticks(x + width*6, adtype)\nbox = ax.get_position()\n# ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])\n\n# ax.legend(loc='upper left', ncols=3)\nplt.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\nax.set_ylim(0.5, 6000)\nplt.yscale('log')\nplt.savefig(\"Dist_feat.png\",bbox_inches='tight')","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:19:17.224147Z","iopub.execute_input":"2023-09-28T10:19:17.224446Z","iopub.status.idle":"2023-09-28T10:19:18.505626Z","shell.execute_reply.started":"2023-09-28T10:19:17.224420Z","shell.execute_reply":"2023-09-28T10:19:18.504462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oridf.drop(columns=[\"c2\",\"c4\",\"connectionType\",\"bidFloorPrice\",\"sentPrice\",\"has_won\"],inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:19:18.506920Z","iopub.execute_input":"2023-09-28T10:19:18.507316Z","iopub.status.idle":"2023-09-28T10:19:19.884962Z","shell.execute_reply.started":"2023-09-28T10:19:18.507285Z","shell.execute_reply":"2023-09-28T10:19:19.883447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Finding the 90% of instances in the \"correctModelName\" feature**","metadata":{}},{"cell_type":"code","source":"relevant_instances(oridf,\"correctModelName\", cutoff=0.90)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:19:19.886530Z","iopub.execute_input":"2023-09-28T10:19:19.886922Z","iopub.status.idle":"2023-09-28T10:19:20.849731Z","shell.execute_reply.started":"2023-09-28T10:19:19.886891Z","shell.execute_reply":"2023-09-28T10:19:20.847665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Appliying some modifications on the make it more robust.**","metadata":{}},{"cell_type":"code","source":"oridf['cMN'] = np.where(oridf.correctModelName.str.contains(\"^SM\"),\"SM\",oridf.correctModelName)\noridf['cMN'] = np.where(oridf.cMN.str.contains(\"^Moto\"),\"Moto\",oridf.cMN)\noridf['cMN'] = np.where(oridf.cMN.str.contains(\"^moto\"),\"Moto\",oridf.cMN)\nd = relevant_instances(oridf,'cMN',cutoff=0.90)\nmeaning = sorted( d , key=lambda x: d[x],reverse=True  )[:6]\noridf['cMN'] = np.where(~oridf.cMN.isin(meaning),'Generic',oridf.cMN)\n\nfig = plt.figure(figsize=(10,5))\nax = fig.add_subplot(121)\nplot_pie_feat(oridf,'correctModelName',ax)\nax.set_title(\"Original\")\nax = fig.add_subplot(122)\nplot_pie_feat(oridf,'cMN',ax)\nax.set_title(\"Modified\")","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:19:20.852337Z","iopub.execute_input":"2023-09-28T10:19:20.853190Z","iopub.status.idle":"2023-09-28T10:19:54.272331Z","shell.execute_reply.started":"2023-09-28T10:19:20.853144Z","shell.execute_reply":"2023-09-28T10:19:54.270816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"relevant_instances(oridf,\"brandName\", cutoff=0.90)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:19:54.274144Z","iopub.execute_input":"2023-09-28T10:19:54.274772Z","iopub.status.idle":"2023-09-28T10:19:54.694377Z","shell.execute_reply.started":"2023-09-28T10:19:54.274722Z","shell.execute_reply":"2023-09-28T10:19:54.692897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = relevant_instances(oridf,'brandName',cutoff=0.90)\nmeaning = sorted( d , key=lambda x: d[x],reverse=True)[:8]\noridf['bN'] = np.where(~oridf.brandName.isin(meaning),'Generic',oridf.brandName)\n\nfig = plt.figure(figsize=(10,5))\nax = fig.add_subplot(121)\nplot_pie_feat(oridf,'brandName',ax)\nax.set_title(\"Original\")\nax = fig.add_subplot(122)\nplot_pie_feat(oridf,'bN',ax)\nax.set_title(\"Modified\")","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:19:54.695639Z","iopub.execute_input":"2023-09-28T10:19:54.695915Z","iopub.status.idle":"2023-09-28T10:19:57.419415Z","shell.execute_reply.started":"2023-09-28T10:19:54.695892Z","shell.execute_reply":"2023-09-28T10:19:57.417822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"relevant_instances(oridf,\"countryCode\", cutoff=0.90)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:19:57.421062Z","iopub.execute_input":"2023-09-28T10:19:57.422006Z","iopub.status.idle":"2023-09-28T10:19:57.868143Z","shell.execute_reply.started":"2023-09-28T10:19:57.421964Z","shell.execute_reply":"2023-09-28T10:19:57.866573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = relevant_instances(oridf,'countryCode',cutoff=0.90)\nmeaning = sorted( d , key=lambda x: d[x],reverse=True)[:12]\noridf['cC'] = np.where(~oridf.countryCode.isin(meaning),'Other',oridf.countryCode)\nfig = plt.figure(figsize=(10,5))\nax = fig.add_subplot(121)\nplot_pie_feat(oridf,'countryCode',ax)\nax.set_title(\"Original\")\nax = fig.add_subplot(122)\nplot_pie_feat(oridf,'cC',ax)\nax.set_title(\"Modified\")","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:19:57.870196Z","iopub.execute_input":"2023-09-28T10:19:57.870623Z","iopub.status.idle":"2023-09-28T10:20:00.909083Z","shell.execute_reply.started":"2023-09-28T10:19:57.870591Z","shell.execute_reply":"2023-09-28T10:20:00.907614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oridf['oAV'] = oridf['osAndVersion'].apply( lambda x : x.split('.')[0])\noridf[\"oAV\"].value_counts()\nfig = plt.figure(figsize=(10,5))\nax = fig.add_subplot(121)\nplot_pie_feat(oridf,'osAndVersion',ax)\nax.set_title(\"Original\")\nax = fig.add_subplot(122)\nplot_pie_feat(oridf,'oAV',ax)\nax.set_title(\"Modified\")","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:20:00.910740Z","iopub.execute_input":"2023-09-28T10:20:00.911374Z","iopub.status.idle":"2023-09-28T10:20:05.614338Z","shell.execute_reply.started":"2023-09-28T10:20:00.911347Z","shell.execute_reply":"2023-09-28T10:20:05.613082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Finding out the relationship between the number of bids and the number of devices. There are about 30K devices with 10 of them having more than 10K bids. Well known users?**","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,5))\nax = fig.add_subplot(121)\ndat = oridf[\"deviceId\"].value_counts().values\nplt.scatter(range(len(dat)),dat,s=5,c='k')\n\ndat = oridf[oridf.unitDisplayType == \"banner\" ][\"deviceId\"].value_counts().values\nplt.scatter(range(len(dat)),dat,c='r',s=5)\ndat = oridf[oridf.unitDisplayType == \"interstitial\" ][\"deviceId\"].value_counts().values\nplt.scatter(range(len(dat)),dat,c='b',s=5)\ndat = oridf[oridf.unitDisplayType == \"rewarded\" ][\"deviceId\"].value_counts().values\nplt.scatter(range(len(dat)),dat,c='g',s=5)\n\nplt.xscale(\"log\")\nplt.yscale(\"log\")\nplt.ylabel(\"Number of bids\")\nplt.xlabel(\"Device number\")\nplt.grid()\n\nax = fig.add_subplot(122)\ndat = oridf[\"deviceId\"].value_counts().values[:50]\nplt.scatter(range(len(dat)),dat,c='k')\nplt.xscale(\"log\")\nplt.yscale(\"log\")\nplt.xlabel(\"Device number\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:20:05.615489Z","iopub.execute_input":"2023-09-28T10:20:05.615779Z","iopub.status.idle":"2023-09-28T10:20:18.669731Z","shell.execute_reply.started":"2023-09-28T10:20:05.615754Z","shell.execute_reply":"2023-09-28T10:20:18.667884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Comparing devices that concentrate the major number of bids to those that appear only have one bid.**","metadata":{}},{"cell_type":"code","source":"device_freq = oridf[\"deviceId\"].value_counts()\ntop10 = list(device_freq[:10].index)\nlast10 = list(device_freq[(device_freq < 4 )].index)\nt10 = oridf[oridf.deviceId.isin(top10)]\nl10 = oridf[oridf.deviceId.isin(last10)]\n\n\nfig = plt.figure()\nax = fig.add_subplot(321)\nplot_pie_feat(t10,\"cC\",ax)\nax = fig.add_subplot(322)\nplot_pie_feat(l10,\"cC\",ax)\nax = fig.add_subplot(323)\nplot_pie_feat(t10,\"bundleId\",ax)\nax = fig.add_subplot(324)\nplot_pie_feat(l10,\"bundleId\",ax)\nax = fig.add_subplot(325)\nplot_pie_feat(t10,\"cMN\",ax)\nax = fig.add_subplot(326)\nplot_pie_feat(l10,\"cMN\",ax)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:20:18.678046Z","iopub.execute_input":"2023-09-28T10:20:18.678446Z","iopub.status.idle":"2023-09-28T10:20:23.316408Z","shell.execute_reply.started":"2023-09-28T10:20:18.678413Z","shell.execute_reply":"2023-09-28T10:20:23.315240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(12,4))\nax = fig.add_subplot(211)\nax.plot(t10.winBid,c='r',label=\"Most Freq. deviceId's\")\nax.set_ylim(0.0,1050)\nax.legend(loc='upper left')\nax.set_ylabel(\"Win Bid  [ $ CPM ]\")\nax = fig.add_subplot(212)\nax.plot(l10.winBid,c='k',label=\"deviceId's Freq <= 3\")\nax.legend(loc='upper left')\nax.set_ylabel(\"Win Bid  [ $ CPM ]\")\nax.set_xlabel(\"Date\")","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:20:23.317926Z","iopub.execute_input":"2023-09-28T10:20:23.318888Z","iopub.status.idle":"2023-09-28T10:20:23.870868Z","shell.execute_reply.started":"2023-09-28T10:20:23.318858Z","shell.execute_reply":"2023-09-28T10:20:23.869317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Interestingly, for well known devices the winbid values are lower than those associted with unknown devices. Why would I bid high to place an Add in unknown device? I am not sure if winning bids on those scencarios can be considered a success (paying high with high uncertainty).**","metadata":{}},{"cell_type":"code","source":"oridf[oridf.deviceId.isin(device_freq[(device_freq < 4 )].index)]","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:20:23.872719Z","iopub.execute_input":"2023-09-28T10:20:23.873118Z","iopub.status.idle":"2023-09-28T10:20:25.337767Z","shell.execute_reply.started":"2023-09-28T10:20:23.873093Z","shell.execute_reply":"2023-09-28T10:20:25.336278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Feature Engineering and Model Building.\n\n**First working only with Banner.**","metadata":{}},{"cell_type":"markdown","source":"Obtaining a subsample of the original dataset.","metadata":{"execution":{"iopub.status.busy":"2023-09-28T08:27:54.847873Z","iopub.execute_input":"2023-09-28T08:27:54.848281Z","iopub.status.idle":"2023-09-28T08:27:54.867612Z","shell.execute_reply.started":"2023-09-28T08:27:54.848250Z","shell.execute_reply":"2023-09-28T08:27:54.866424Z"}}},{"cell_type":"code","source":"columns_to_reduce = [ 'cMN', 'bN', 'cC', 'oAV']\naddtype = \"banner\"\ndict_of_cols = get_col_dicts(oridf[oridf.unitDisplayType==addtype].copy(),\n                             columns_to_reduce,\n                             200000,\n                             cutoff=0.99)\nseen_ndx = ndx_to_work(dict_of_cols)\ntraining_set = oridf[oridf.unitDisplayType==addtype].copy().loc[list(seen_ndx)]\ndata_add = training_set.copy()\n\ndel training_set\n\ndata_add = data_add.drop(columns=['unitDisplayType','brandName',\n                                        'correctModelName', 'countryCode','osAndVersion'])","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:20:25.339486Z","iopub.execute_input":"2023-09-28T10:20:25.339782Z","iopub.status.idle":"2023-09-28T10:21:03.524257Z","shell.execute_reply.started":"2023-09-28T10:20:25.339759Z","shell.execute_reply":"2023-09-28T10:21:03.522592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total=sum(data_add.winBid)\nprice_cutoff = 2.5\nc = 0\nfor n,i in enumerate(sorted(data_add.winBid)):\n    c += i/total\n    if (c > 0.50) & (i > price_cutoff):\n        print(\"For %s cases, \"%n+\\\n            \"which represent the %.2f%% of the market,\\n\"%(c*100)+\n            \"the Add's price lays below $%.2f. \"%i+\n            \"These cases constitute the %.2f%% from the total number of bids.\\n\"%(100*n/len(data_add)))\n        break","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:21:03.525939Z","iopub.execute_input":"2023-09-28T10:21:03.526385Z","iopub.status.idle":"2023-09-28T10:21:04.120651Z","shell.execute_reply.started":"2023-09-28T10:21:03.526350Z","shell.execute_reply":"2023-09-28T10:21:04.118920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oo = data_add.groupby(by=\"deviceId\")\nup_cf_counts = list(map(lambda x: (x,sum(oo.get_group(x).winBid > price_cutoff )),data_add[\"deviceId\"].unique()))\ndel oo\nucfc = { c[0]:c[1] for c in up_cf_counts }\ntemp = data_add[\"deviceId\"].value_counts()\nmm = sorted([[i,temp.loc[i],ucfc[i],ucfc[i]/temp.loc[i]] for i in temp.index],key= lambda x: x[2],reverse=True )\ndel temp\nuser_add_df = pd.DataFrame([i[1:] for i in mm],\n                           columns=[\"N_appear\",\"N_above_50\",\"above2apper\"],\n                           index=[i[0] for i in mm])\ndel mm","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:21:04.122949Z","iopub.execute_input":"2023-09-28T10:21:04.123426Z","iopub.status.idle":"2023-09-28T10:21:13.346567Z","shell.execute_reply.started":"2023-09-28T10:21:04.123386Z","shell.execute_reply":"2023-09-28T10:21:13.344684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = sorted(user_add_df[[\"above2apper\",\"N_appear\"]].to_numpy(),\n              key=lambda x: (x[0],x[1]),\n              reverse=False)\ndata = np.array(data)\nprint(data.max(axis=0))\ncmap = matplotlib.colormaps['OrRd']\nnormalize = matplotlib.colors.Normalize(vmin=1, vmax=10)\n\nfig = plt.figure(figsize=(18,3))\nax = fig.add_subplot(111)\nim = ax.scatter(range(len(data)),\n           [ i[0] for i in data],\n            c=[ i[1] for i in data],\n            cmap=cmap,norm=normalize,\n            alpha=0.6,s=5,edgecolors=None)\ncbar = fig.colorbar(im, ax=ax)\ncbar.ax.get_yaxis().set_ticks([1,10],labels=[\"1\",\">10\"])\ncbar.ax.set_ylabel('# total bids', rotation=90)\nax.set_xlim(16000,22000)\nax.set_ylabel(\"#_of_expensive_bids/#_total_bids\")\nax.set_xlabel(\"DeviceId Rank\")\nax.set_title(\"Devices expensive tendency (%s price > $%s )\"%(addtype,price_cutoff))\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:21:13.348741Z","iopub.execute_input":"2023-09-28T10:21:13.349489Z","iopub.status.idle":"2023-09-28T10:21:14.220732Z","shell.execute_reply.started":"2023-09-28T10:21:13.349436Z","shell.execute_reply":"2023-09-28T10:21:14.219105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_add_df['myk'] = 0\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper==0)),1,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper>0)&(user_add_df.above2apper<0.5)),2,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper>0)&(user_add_df.above2apper<0.5)),3,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper>=0.5)&(user_add_df.above2apper<1.)),4,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper>=0.5)&(user_add_df.above2apper<1.)),5,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper==1.)),6,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper==1.)),7,user_add_df['myk'])","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:21:14.222209Z","iopub.execute_input":"2023-09-28T10:21:14.222556Z","iopub.status.idle":"2023-09-28T10:21:14.238770Z","shell.execute_reply.started":"2023-09-28T10:21:14.222530Z","shell.execute_reply":"2023-09-28T10:21:14.237367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wk_add_df = pd.merge(data_add,user_add_df,how='left',left_on=\"deviceId\",right_index=True)\ndel data_add","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:21:14.240243Z","iopub.execute_input":"2023-09-28T10:21:14.240532Z","iopub.status.idle":"2023-09-28T10:21:14.742280Z","shell.execute_reply.started":"2023-09-28T10:21:14.240507Z","shell.execute_reply":"2023-09-28T10:21:14.740295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(5,5))\nax = fig.add_subplot(111)\nax.scatter(wk_add_df[\"N_appear\"],\n            wk_add_df[\"N_above_50\"],alpha=0.1,color=\"k\",s=5)\nplt.ylabel(\"# of time the price was > $%s \"%price_cutoff)\nplt.xlabel(\"# of bids\")\nplt.title(\"#_bids vs #_expensive_bids per user\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:21:14.745962Z","iopub.execute_input":"2023-09-28T10:21:14.746637Z","iopub.status.idle":"2023-09-28T10:21:15.642676Z","shell.execute_reply.started":"2023-09-28T10:21:14.746602Z","shell.execute_reply":"2023-09-28T10:21:15.641541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,4))\nax = fig.add_subplot(121)\nplot_pie_feat(wk_add_df,'myk',ax)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:21:15.643819Z","iopub.execute_input":"2023-09-28T10:21:15.644551Z","iopub.status.idle":"2023-09-28T10:21:15.777167Z","shell.execute_reply.started":"2023-09-28T10:21:15.644524Z","shell.execute_reply":"2023-09-28T10:21:15.776242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wk_add_df[\"ndx\"] = range(wk_add_df.shape[0])\ndef rowFunc(row):\n    return pd.Series(row.values,index=['m%s'%i for i in range(len(row),0,-1)])\n\nlookmin = 6\nnkluster = len(wk_add_df.myk.value_counts())\ncm = [ 'm%s'%i for i in range(lookmin,0,-1) ]\ntime_ban_df = pd.DataFrame(columns = cm)\nfor k in range(nkluster):\n    rr = wk_add_df[wk_add_df.myk == k]\n    ff = rr.winBid.resample('1min').mean()\n    pp = rr.apply(lambda x : rowFunc(ff[x.name-pd.Timedelta(minutes=lookmin):x.name-pd.Timedelta(minutes=1)]),axis=1)\n    time_ban_df= pd.concat([time_ban_df,pd.concat([pp,rr.ndx],axis=1)],axis=0)\ndel rr, ff, pp","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:21:15.778327Z","iopub.execute_input":"2023-09-28T10:21:15.779165Z","iopub.status.idle":"2023-09-28T10:29:04.333103Z","shell.execute_reply.started":"2023-09-28T10:21:15.779130Z","shell.execute_reply":"2023-09-28T10:29:04.331979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_classes = {\"categorical\": [],\n                    \"categorical_market\": ['bundleId', 'appVersion', 'deviceId',\n                                            'c1', 'c3','size',\n                                           'mediationProviderVersion','cMN', 'bN', 'cC', 'oAV'\n                                           ],\n                    \"ready\": [\"hour\",\"wday\", 'N_appear', 'N_above_50', 'above2apper','m6','m5', 'm4', 'm3', 'm2', 'm1'],\n                   \"monitor\": [\"winBid\"]}\n\nfeat_order = [\"cC\",\"oAV\",\"bN\",\n              \"size\",\n              \"bundleId\",\"c1\",\"c3\",\"deviceId\"]+['N_appear', 'N_above_50', 'above2apper',\n                                                'm6','m5', 'm4', 'm3', 'm2', 'm1']+[\"winBid\"]","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:04.334517Z","iopub.execute_input":"2023-09-28T10:29:04.335829Z","iopub.status.idle":"2023-09-28T10:29:04.342968Z","shell.execute_reply.started":"2023-09-28T10:29:04.335773Z","shell.execute_reply":"2023-09-28T10:29:04.341638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_ban_df['ndx'] = time_ban_df['ndx'].astype(int)\ntime_ban_df.set_index(\"ndx\",inplace=True)\nwk_add_df = pd.merge(wk_add_df,time_ban_df,how='left',left_on=\"ndx\", right_index=True)\nwk_add_df.fillna(0, inplace=True)\nwk_add_df['eventTimestamp'] = wk_add_df.index\n\nencoders_ban = get_df_encoders(wk_add_df,feature_classes)\n\nori_trans_ban = transform_df(wk_add_df.copy(),encoders_ban,feature_classes)\n\nori_trans_ban = ori_trans_ban[feat_order]","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:04.344721Z","iopub.execute_input":"2023-09-28T10:29:04.345056Z","iopub.status.idle":"2023-09-28T10:29:12.170108Z","shell.execute_reply.started":"2023-09-28T10:29:04.345012Z","shell.execute_reply":"2023-09-28T10:29:12.168691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_trai , temp_trans , target_trai , target_temp = train_test_split(ori_trans_ban.drop(columns=[\"winBid\"]),\n                                                   ori_trans_ban[\"winBid\"],test_size=0.2,random_state=1)\nreg_vali , reg_test , target_vali , target_test = train_test_split(temp_trans,target_temp,test_size=0.5,random_state=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:12.172178Z","iopub.execute_input":"2023-09-28T10:29:12.172762Z","iopub.status.idle":"2023-09-28T10:29:12.379211Z","shell.execute_reply.started":"2023-09-28T10:29:12.172736Z","shell.execute_reply":"2023-09-28T10:29:12.378166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_reg_train = lgb.Dataset(reg_trai, target_trai )\nlgb_reg_eval = lgb.Dataset(reg_vali, target_vali, reference=lgb_reg_train)\n\nparams_reg = {\n    'task': 'train', \n    'boosting': 'gbdt',\n    'objective':'regression',    \n    'num_leaves': int((2**7)*0.75),\n    'max_depth' : 7,\n    'subsample_freq':5,\n    'subsample' : 0.8,\n    'min_data_in_leaf': 50,\n     'min_child_weight': 10.,\n#    'colsample_bytree': 0.8,\n    'learning_rate': 0.1,\n    'metric':{'rmse'},\n    'verbose': -1,\n    'num_iterations':20000,\n}\n\nmodel_reg = lgb.train(params_reg,\n                 train_set=lgb_reg_train,\n                 valid_sets=lgb_reg_eval,\n                 callbacks=[early_stopping(5), log_evaluation(100)])\n\ntarget_test_pred = eval_model(model_reg,reg_test,corr_type='nog')\nrmse_test = mean_squared_error(target_test_pred,target_test)**(0.5)\nprint(\"RMSE test : %.2f\" % rmse_test)\n\ntarget_vali_pred = eval_model(model_reg,reg_vali,corr_type='nog')\nrmse_vali = mean_squared_error(target_vali_pred,target_vali)**(0.5)\nprint(\"RMSE vali : %.2f\" % rmse_vali)\n\ntarget_trai_pred = eval_model(model_reg,reg_trai,corr_type='nog')\nrmse_trai = mean_squared_error(target_trai_pred,target_trai)**(0.5)\nprint(\"RMSE train : %.2f\" % rmse_trai)\n\nfig = plt.figure(figsize=(11,5))\nax = fig.add_subplot(121)\nax.plot(np.linspace(0.01,100,10),np.linspace(0.01,100,10),'k')\nax.scatter(target_vali_pred,target_vali, alpha=0.1)\nax.set_title(\"Validation Data\")\nplt.xlabel(\"Predicted Win Bid [ $ CPM ]\")\nplt.ylabel(\"Real Win Bid [ $ CPM ]\")\nplt.yscale(\"log\")\nplt.xscale(\"log\")\nplt.grid()\n\nax = fig.add_subplot(122)\nax.plot(np.linspace(0.01,100,10),np.linspace(0.01,100,10),'k')\nax.scatter(target_trai_pred,target_trai, alpha=0.1)\nax.set_title(\"Training Data\")\nplt.xlabel(\"Predicted Win Bid [ $ CPM ]\")\nplt.ylabel(\"Real Win Bid [ $ CPM ]\")\n# plt.yscale(\"log\")\n# plt.xscale(\"log\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:12.380419Z","iopub.execute_input":"2023-09-28T10:29:12.381596Z","iopub.status.idle":"2023-09-28T10:29:28.320969Z","shell.execute_reply.started":"2023-09-28T10:29:12.381556Z","shell.execute_reply":"2023-09-28T10:29:28.319627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Now working only with Interstitial.**","metadata":{}},{"cell_type":"code","source":"addtype = \"interstitial\"\ndata_add = oridf[oridf.unitDisplayType==addtype].copy()\ndata_add.drop(columns=['unitDisplayType','brandName',\n                        'correctModelName', 'countryCode','osAndVersion'],inplace=True)\nprint(\"Shape of data_inters :\",data_add.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:28.322282Z","iopub.execute_input":"2023-09-28T10:29:28.323272Z","iopub.status.idle":"2023-09-28T10:29:29.203933Z","shell.execute_reply.started":"2023-09-28T10:29:28.323241Z","shell.execute_reply":"2023-09-28T10:29:29.203224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total=sum(data_add.winBid)\nprice_cutoff = 50\nc = 0\nfor n,i in enumerate(sorted(data_add.winBid)):\n    c += i/total\n    if (c > 0.50) & (i > price_cutoff):\n        print(\"For %s cases, \"%n+\\\n            \"which represent the %.2f%% of the market,\\n\"%(c*100)+\n            \"the Add's price lays below $%.2f. \"%i+\n            \"These cases constitute the %.2f%% from the total number of bids.\\n\"%(100*n/len(data_add)))\n        break","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:29.204922Z","iopub.execute_input":"2023-09-28T10:29:29.205758Z","iopub.status.idle":"2023-09-28T10:29:29.783510Z","shell.execute_reply.started":"2023-09-28T10:29:29.205731Z","shell.execute_reply":"2023-09-28T10:29:29.782217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oo = data_add.groupby(by=\"deviceId\")\nup_cf_counts = list(map(lambda x: (x,sum(oo.get_group(x).winBid > price_cutoff )),data_add[\"deviceId\"].unique()))\ndel oo\nucfc = { c[0]:c[1] for c in up_cf_counts }\ntemp = data_add[\"deviceId\"].value_counts()\nmm = sorted([[i,temp.loc[i],ucfc[i],ucfc[i]/temp.loc[i]] for i in temp.index],key= lambda x: x[2],reverse=True )\ndel temp\nuser_add_df = pd.DataFrame([i[1:] for i in mm],\n                           columns=[\"N_appear\",\"N_above_50\",\"above2apper\"],\n                           index=[i[0] for i in mm])\ndel mm","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:29.785532Z","iopub.execute_input":"2023-09-28T10:29:29.785911Z","iopub.status.idle":"2023-09-28T10:29:42.697483Z","shell.execute_reply.started":"2023-09-28T10:29:29.785880Z","shell.execute_reply":"2023-09-28T10:29:42.696136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = sorted(user_add_df[[\"above2apper\",\"N_appear\"]].to_numpy(),\n              key=lambda x: (x[0],x[1]),\n              reverse=False)\ndata = np.array(data)\nprint(data.max(axis=0))\ncmap = matplotlib.colormaps['OrRd']\nnormalize = matplotlib.colors.Normalize(vmin=1, vmax=10)\n\nfig = plt.figure(figsize=(18,3))\nax = fig.add_subplot(111)\nim = ax.scatter(range(len(data)),\n           [ i[0] for i in data],\n            c=[ i[1] for i in data],\n            cmap=cmap,norm=normalize,\n            alpha=0.6,s=5,edgecolors=None)\ncbar = fig.colorbar(im, ax=ax)\ncbar.ax.get_yaxis().set_ticks([1,10],labels=[\"1\",\">10\"])\ncbar.ax.set_ylabel('# total bids', rotation=90)\nax.set_xlim(20000,35000)\nax.set_ylabel(\"#_of_expensive_bids/#_total_bids\")\nax.set_xlabel(\"DeviceId Rank\")\nax.set_title(\"Devices expensive tendency (%s price > $%s )\"%(addtype,price_cutoff))\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:42.699314Z","iopub.execute_input":"2023-09-28T10:29:42.699759Z","iopub.status.idle":"2023-09-28T10:29:43.754712Z","shell.execute_reply.started":"2023-09-28T10:29:42.699726Z","shell.execute_reply":"2023-09-28T10:29:43.753785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_add_df['myk'] = 0\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper==0)),1,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper>0)&(user_add_df.above2apper<0.5)),2,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper>0)&(user_add_df.above2apper<0.5)),3,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper>=0.5)&(user_add_df.above2apper<1.)),4,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper>=0.5)&(user_add_df.above2apper<1.)),5,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper==1.)),6,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper==1.)),7,user_add_df['myk'])","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:43.755993Z","iopub.execute_input":"2023-09-28T10:29:43.756279Z","iopub.status.idle":"2023-09-28T10:29:43.772826Z","shell.execute_reply.started":"2023-09-28T10:29:43.756256Z","shell.execute_reply":"2023-09-28T10:29:43.771329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wk_add_df = pd.merge(data_add,user_add_df,how='left',left_on=\"deviceId\",right_index=True)\ndel data_add","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:43.774464Z","iopub.execute_input":"2023-09-28T10:29:43.774753Z","iopub.status.idle":"2023-09-28T10:29:44.180319Z","shell.execute_reply.started":"2023-09-28T10:29:43.774731Z","shell.execute_reply":"2023-09-28T10:29:44.178609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(5,5))\nax = fig.add_subplot(111)\nax.scatter(wk_add_df[\"N_appear\"],\n            wk_add_df[\"N_above_50\"],alpha=0.1,color=\"k\",s=5)\nplt.ylabel(\"# of time the price was > $%s \"%price_cutoff)\nplt.xlabel(\"# of bids\")\nplt.title(\"#_bids vs #_expensive_bids per user\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:44.182060Z","iopub.execute_input":"2023-09-28T10:29:44.182457Z","iopub.status.idle":"2023-09-28T10:29:44.964587Z","shell.execute_reply.started":"2023-09-28T10:29:44.182426Z","shell.execute_reply":"2023-09-28T10:29:44.963678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,4))\nax = fig.add_subplot(121)\nplot_pie_feat(wk_add_df,'myk',ax)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:44.966112Z","iopub.execute_input":"2023-09-28T10:29:44.966520Z","iopub.status.idle":"2023-09-28T10:29:45.088037Z","shell.execute_reply.started":"2023-09-28T10:29:44.966496Z","shell.execute_reply":"2023-09-28T10:29:45.086930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wk_add_df[\"ndx\"] = range(wk_add_df.shape[0])\ndef rowFunc(row):\n    return pd.Series(row.values,index=['m%s'%i for i in range(len(row),0,-1)])\n\nlookmin = 6\nnkluster = len(wk_add_df.myk.value_counts())\ncm = [ 'm%s'%i for i in range(lookmin,0,-1) ]\ntime_int_df = pd.DataFrame(columns = cm)\nfor k in range(nkluster):\n    rr = wk_add_df[wk_add_df.myk == k]\n    ff = rr.winBid.resample('1min').mean()\n    pp = rr.apply(lambda x : rowFunc(ff[x.name-pd.Timedelta(minutes=lookmin):x.name-pd.Timedelta(minutes=1)]),axis=1)\n    time_int_df= pd.concat([time_int_df,pd.concat([pp,rr.ndx],axis=1)],axis=0)\ndel rr, ff, pp","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:29:45.089563Z","iopub.execute_input":"2023-09-28T10:29:45.089921Z","iopub.status.idle":"2023-09-28T10:36:50.411955Z","shell.execute_reply.started":"2023-09-28T10:29:45.089889Z","shell.execute_reply":"2023-09-28T10:36:50.410738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_int_df['ndx'] = time_int_df['ndx'].astype(int)\ntime_int_df.set_index(\"ndx\",inplace=True)\nwk_add_df = pd.merge(wk_add_df,time_int_df,how='left',left_on=\"ndx\", right_index=True)\nwk_add_df.fillna(0, inplace=True)\nwk_add_df['eventTimestamp'] = wk_add_df.index\n\nencoders_int = get_df_encoders(wk_add_df,feature_classes)\n\nori_trans_int = transform_df(wk_add_df.copy(),encoders_int,feature_classes)\n\nori_trans_int = ori_trans_int[feat_order]","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:36:50.413329Z","iopub.execute_input":"2023-09-28T10:36:50.413645Z","iopub.status.idle":"2023-09-28T10:36:57.472430Z","shell.execute_reply.started":"2023-09-28T10:36:50.413620Z","shell.execute_reply":"2023-09-28T10:36:57.470797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_trai , temp_trans , target_trai , target_temp = train_test_split(ori_trans_int.drop(columns=[\"winBid\"]),\n                                                   ori_trans_int[\"winBid\"],test_size=0.2,random_state=1)\nreg_vali , reg_test , target_vali , target_test = train_test_split(temp_trans,target_temp,test_size=0.5,random_state=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:36:57.474132Z","iopub.execute_input":"2023-09-28T10:36:57.474553Z","iopub.status.idle":"2023-09-28T10:36:57.666817Z","shell.execute_reply.started":"2023-09-28T10:36:57.474525Z","shell.execute_reply":"2023-09-28T10:36:57.665351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_reg_train = lgb.Dataset(reg_trai, target_trai )\nlgb_reg_eval = lgb.Dataset(reg_vali, target_vali, reference=lgb_reg_train)\n\nparams_reg = {\n    'task': 'train', \n    'boosting': 'gbdt',\n    'objective':'regression',    \n    'num_leaves': int((2**5)*0.75),\n    'max_depth' : 5,\n    'subsample_freq':5,\n    'subsample' : 0.8,\n    'min_data_in_leaf': 50,\n     'min_child_weight': 10.,\n#    'colsample_bytree': 0.8,\n    'learning_rate': 0.1,\n    'metric':{'rmse'},\n    'verbose': -1,\n    'num_iterations':20000,\n}\n\nmodel_reg = lgb.train(params_reg,\n                 train_set=lgb_reg_train,\n                 valid_sets=lgb_reg_eval,\n                 callbacks=[early_stopping(5), log_evaluation(100)])\n\ntarget_test_pred = eval_model(model_reg,reg_test,corr_type='nog')\nrmse_test = mean_squared_error(target_test_pred,target_test)**(0.5)\nprint(\"RMSE test : %.2f\" % rmse_test)\n\ntarget_vali_pred = eval_model(model_reg,reg_vali,corr_type='nog')\nrmse_vali = mean_squared_error(target_vali_pred,target_vali)**(0.5)\nprint(\"RMSE vali : %.2f\" % rmse_vali)\n\ntarget_trai_pred = eval_model(model_reg,reg_trai,corr_type='nog')\nrmse_trai = mean_squared_error(target_trai_pred,target_trai)**(0.5)\nprint(\"RMSE train : %.2f\" % rmse_trai)\n\nfig = plt.figure(figsize=(11,5))\nax = fig.add_subplot(121)\nax.plot(np.linspace(0.01,1000,10),np.linspace(0.01,1000,10),'k')\nax.scatter(target_vali_pred,target_vali, alpha=0.1)\nax.set_title(\"Validation Data\")\nplt.xlabel(\"Predicted Win Bid [ $ CPM ]\")\nplt.ylabel(\"Real Win Bid [ $ CPM ]\")\nplt.yscale(\"log\")\nplt.xscale(\"log\")\nplt.grid()\n\nax = fig.add_subplot(122)\nax.plot(np.linspace(0.01,1000,10),np.linspace(0.01,1000,10),'k')\nax.scatter(target_trai_pred,target_trai, alpha=0.1)\nax.set_title(\"Training Data\")\nplt.xlabel(\"Predicted Win Bid [ $ CPM ]\")\nplt.ylabel(\"Real Win Bid [ $ CPM ]\")\n# plt.yscale(\"log\")\n# plt.xscale(\"log\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:36:57.669063Z","iopub.execute_input":"2023-09-28T10:36:57.669502Z","iopub.status.idle":"2023-09-28T10:37:22.986835Z","shell.execute_reply.started":"2023-09-28T10:36:57.669466Z","shell.execute_reply":"2023-09-28T10:37:22.986142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Working with rewarded**","metadata":{}},{"cell_type":"code","source":"addtype = \"rewarded\"\ndata_add = oridf[oridf.unitDisplayType==addtype].copy()\ndata_add.drop(columns=['unitDisplayType','brandName',\n                        'correctModelName', 'countryCode','osAndVersion'],inplace=True)\nprint(\"Shape of data_inters :\",data_add.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:22.987736Z","iopub.execute_input":"2023-09-28T10:37:22.989214Z","iopub.status.idle":"2023-09-28T10:37:24.020929Z","shell.execute_reply.started":"2023-09-28T10:37:22.989156Z","shell.execute_reply":"2023-09-28T10:37:24.019566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total=sum(data_add.winBid)\nprice_cutoff = 50\nc = 0\nfor n,i in enumerate(sorted(data_add.winBid)):\n    c += i/total\n    if (c > 0.50) & (i > price_cutoff):\n        print(\"For %s cases, \"%n+\\\n            \"which represent the %.2f%% of the market,\\n\"%(c*100)+\n            \"the Add's price lays below $%.2f. \"%i+\n            \"These cases constitute the %.2f%% from the total number of bids.\\n\"%(100*n/len(data_add)))\n        break","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:24.021919Z","iopub.execute_input":"2023-09-28T10:37:24.022200Z","iopub.status.idle":"2023-09-28T10:37:24.740125Z","shell.execute_reply.started":"2023-09-28T10:37:24.022177Z","shell.execute_reply":"2023-09-28T10:37:24.739334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oo = data_add.groupby(by=\"deviceId\")\nup_cf_counts = list(map(lambda x: (x,sum(oo.get_group(x).winBid > price_cutoff )),data_add[\"deviceId\"].unique()))\ndel oo\nucfc = { c[0]:c[1] for c in up_cf_counts }\ntemp = data_add[\"deviceId\"].value_counts()\nmm = sorted([[i,temp.loc[i],ucfc[i],ucfc[i]/temp.loc[i]] for i in temp.index],key= lambda x: x[2],reverse=True )\ndel temp\nuser_add_df = pd.DataFrame([i[1:] for i in mm],\n                           columns=[\"N_appear\",\"N_above_50\",\"above2apper\"],\n                           index=[i[0] for i in mm])\ndel mm","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:24.741099Z","iopub.execute_input":"2023-09-28T10:37:24.742011Z","iopub.status.idle":"2023-09-28T10:37:38.020296Z","shell.execute_reply.started":"2023-09-28T10:37:24.741949Z","shell.execute_reply":"2023-09-28T10:37:38.018653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = sorted(user_add_df[[\"above2apper\",\"N_appear\"]].to_numpy(),\n              key=lambda x: (x[0],x[1]),\n              reverse=False)\ndata = np.array(data)\nprint(data.max(axis=0))\ncmap = matplotlib.colormaps['OrRd']\nnormalize = matplotlib.colors.Normalize(vmin=1, vmax=10)\n\nfig = plt.figure(figsize=(18,3))\nax = fig.add_subplot(111)\nim = ax.scatter(range(len(data)),\n           [ i[0] for i in data],\n            c=[ i[1] for i in data],\n            cmap=cmap,norm=normalize,\n            alpha=0.6,s=5,edgecolors=None)\ncbar = fig.colorbar(im, ax=ax)\ncbar.ax.get_yaxis().set_ticks([1,10],labels=[\"1\",\">10\"])\ncbar.ax.set_ylabel('# total bids', rotation=90)\nax.set_xlim(20000,35000)\nax.set_ylabel(\"#_of_expensive_bids/#_total_bids\")\nax.set_xlabel(\"DeviceId Rank\")\nax.set_title(\"Devices expensive tendency (%s price > $%s )\"%(addtype,price_cutoff))\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:38.021826Z","iopub.execute_input":"2023-09-28T10:37:38.022269Z","iopub.status.idle":"2023-09-28T10:37:39.110095Z","shell.execute_reply.started":"2023-09-28T10:37:38.022237Z","shell.execute_reply":"2023-09-28T10:37:39.109161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_add_df['myk'] = 0\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper==0)),1,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper>0)&(user_add_df.above2apper<0.5)),2,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper>0)&(user_add_df.above2apper<0.5)),3,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper>=0.5)&(user_add_df.above2apper<1.)),4,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper>=0.5)&(user_add_df.above2apper<1.)),5,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear<=3)&(user_add_df.above2apper==1.)),6,user_add_df['myk'])\nuser_add_df['myk'] = np.where(((user_add_df.N_appear>3)&(user_add_df.above2apper==1.)),7,user_add_df['myk'])","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:39.111016Z","iopub.execute_input":"2023-09-28T10:37:39.111458Z","iopub.status.idle":"2023-09-28T10:37:39.127404Z","shell.execute_reply.started":"2023-09-28T10:37:39.111434Z","shell.execute_reply":"2023-09-28T10:37:39.126085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wk_add_df = pd.merge(data_add,user_add_df,how='left',left_on=\"deviceId\",right_index=True)\ndel data_add","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:39.128971Z","iopub.execute_input":"2023-09-28T10:37:39.129308Z","iopub.status.idle":"2023-09-28T10:37:39.665701Z","shell.execute_reply.started":"2023-09-28T10:37:39.129281Z","shell.execute_reply":"2023-09-28T10:37:39.664156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(5,5))\nax = fig.add_subplot(111)\nax.scatter(wk_add_df[\"N_appear\"],\n            wk_add_df[\"N_above_50\"],alpha=0.1,color=\"k\",s=5)\nplt.ylabel(\"# of time the price was > $%s \"%price_cutoff)\nplt.xlabel(\"# of bids\")\nplt.title(\"#_bids vs #_expensive_bids per user\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:39.667190Z","iopub.execute_input":"2023-09-28T10:37:39.667544Z","iopub.status.idle":"2023-09-28T10:37:40.605211Z","shell.execute_reply.started":"2023-09-28T10:37:39.667514Z","shell.execute_reply":"2023-09-28T10:37:40.604470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,4))\nax = fig.add_subplot(121)\nplot_pie_feat(wk_add_df,'myk',ax)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:40.606141Z","iopub.execute_input":"2023-09-28T10:37:40.607304Z","iopub.status.idle":"2023-09-28T10:37:40.735762Z","shell.execute_reply.started":"2023-09-28T10:37:40.607274Z","shell.execute_reply":"2023-09-28T10:37:40.734766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wk_add_df[\"ndx\"] = range(wk_add_df.shape[0])\ndef rowFunc(row):\n    return pd.Series(row.values,index=['m%s'%i for i in range(len(row),0,-1)])\n\nlookmin = 6\nnkluster = len(wk_add_df.myk.value_counts())\ncm = [ 'm%s'%i for i in range(lookmin,0,-1) ]\ntime_rew_df = pd.DataFrame(columns = cm)\nfor k in range(nkluster):\n    rr = wk_add_df[wk_add_df.myk == k]\n    ff = rr.winBid.resample('1min').mean()\n    pp = rr.apply(lambda x : rowFunc(ff[x.name-pd.Timedelta(minutes=lookmin):x.name-pd.Timedelta(minutes=1)]),axis=1)\n    time_rew_df= pd.concat([time_rew_df,pd.concat([pp,rr.ndx],axis=1)],axis=0)\ndel rr, ff, pp","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:37:40.737123Z","iopub.execute_input":"2023-09-28T10:37:40.737454Z","iopub.status.idle":"2023-09-28T10:46:27.101373Z","shell.execute_reply.started":"2023-09-28T10:37:40.737425Z","shell.execute_reply":"2023-09-28T10:46:27.099788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_rew_df['ndx'] = time_rew_df['ndx'].astype(int)\ntime_rew_df.set_index(\"ndx\",inplace=True)\nwk_add_df = pd.merge(wk_add_df,time_rew_df,how='left',left_on=\"ndx\", right_index=True)\nwk_add_df.fillna(0, inplace=True)\nwk_add_df['eventTimestamp'] = wk_add_df.index\n\nencoders_rew = get_df_encoders(wk_add_df,feature_classes)\n\nori_trans_rew = transform_df(wk_add_df.copy(),encoders_rew,feature_classes)\n\nori_trans_rew = ori_trans_rew[feat_order]","metadata":{"execution":{"iopub.status.busy":"2023-09-28T11:36:01.050110Z","iopub.execute_input":"2023-09-28T11:36:01.051435Z","iopub.status.idle":"2023-09-28T11:36:11.086618Z","shell.execute_reply.started":"2023-09-28T11:36:01.051287Z","shell.execute_reply":"2023-09-28T11:36:11.084733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_trai , temp_trans , target_trai , target_temp = train_test_split(ori_trans_rew.drop(columns=[\"winBid\"]),\n                                                   ori_trans_rew[\"winBid\"],test_size=0.2,random_state=1)\nreg_vali , reg_test , target_vali , target_test = train_test_split(temp_trans,target_temp,test_size=0.5,random_state=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-28T11:36:22.685379Z","iopub.execute_input":"2023-09-28T11:36:22.685726Z","iopub.status.idle":"2023-09-28T11:36:22.967720Z","shell.execute_reply.started":"2023-09-28T11:36:22.685693Z","shell.execute_reply":"2023-09-28T11:36:22.966251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_reg_train = lgb.Dataset(reg_trai, target_trai )\nlgb_reg_eval = lgb.Dataset(reg_vali, target_vali, reference=lgb_reg_train)\n\nparams_reg = {\n    'task': 'train', \n    'boosting': 'gbdt',\n    'objective':'regression',    \n    'num_leaves': int((2**7)*0.75),\n    'max_depth' : 7,\n    'subsample_freq':5,\n    'subsample' : 0.8,\n    'min_data_in_leaf': 50,\n     'min_child_weight': 10.,\n#    'colsample_bytree': 0.8,\n    'learning_rate': 0.1,\n    'metric':{'rmse'},\n    'verbose': -1,\n    'num_iterations':20000,\n}\n\nmodel_reg = lgb.train(params_reg,\n                 train_set=lgb_reg_train,\n                 valid_sets=lgb_reg_eval,\n                 callbacks=[early_stopping(5), log_evaluation(100)])\n\ntarget_test_pred = eval_model(model_reg,reg_test,corr_type='nog')\nrmse_test = mean_squared_error(target_test_pred,target_test)**(0.5)\nprint(\"RMSE test : %.2f\" % rmse_test)\n\ntarget_vali_pred = eval_model(model_reg,reg_vali,corr_type='nog')\nrmse_vali = mean_squared_error(target_vali_pred,target_vali)**(0.5)\nprint(\"RMSE vali : %.2f\" % rmse_vali)\n\ntarget_trai_pred = eval_model(model_reg,reg_trai,corr_type='nog')\nrmse_trai = mean_squared_error(target_trai_pred,target_trai)**(0.5)\nprint(\"RMSE train : %.2f\" % rmse_trai)\n\nfig = plt.figure(figsize=(11,5))\nax = fig.add_subplot(121)\nax.plot(np.linspace(0.01,1000,10),np.linspace(0.01,1000,10),'k')\nax.scatter(target_vali_pred,target_vali, alpha=0.1)\nax.set_title(\"Validation Data\")\nplt.xlabel(\"Predicted Win Bid [ $ CPM ]\")\nplt.ylabel(\"Real Win Bid [ $ CPM ]\")\nplt.yscale(\"log\")\nplt.xscale(\"log\")\nplt.grid()\n\nax = fig.add_subplot(122)\nax.plot(np.linspace(0.01,1000,10),np.linspace(0.01,1000,10),'k')\nax.scatter(target_trai_pred,target_trai, alpha=0.1)\nax.set_title(\"Training Data\")\nplt.xlabel(\"Predicted Win Bid [ $ CPM ]\")\nplt.ylabel(\"Real Win Bid [ $ CPM ]\")\n# plt.yscale(\"log\")\n# plt.xscale(\"log\")\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2023-09-28T11:36:27.075579Z","iopub.execute_input":"2023-09-28T11:36:27.075999Z","iopub.status.idle":"2023-09-28T11:36:48.012453Z","shell.execute_reply.started":"2023-09-28T11:36:27.075965Z","shell.execute_reply":"2023-09-28T11:36:48.011215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Modeling and Evaluation\n**First a ensamble survival models is going to be implemented. Please see: https://scikit-survival.readthedocs.io/en/stable/user_guide/boosting.html and https://doi.org/10.1016/j.knosys.2022.109248.**","metadata":{}}]}