{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Neccessary files and  Modules\n## Aggregate data step","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd\nimport gc# data processing, CSV file I/O (e.g. pd.read_csv)\ntrain=pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/train_data.csv')\ndf  = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/test_data.csv')\ndisplay(df.isnull().sum())\n\n# AGGRERATE COLUMNS\ndef agg_functions(df1):\n    features=['c1','deviceId','unitDisplayType','bundleId']\n    global train\n    for x in features:\n        t=train.groupby(x)['winBid'].agg(['std','max','min','mean','median','var','sem'])\n        t.columns=[x+c+'_bidWin' for c in t.columns]\n        t=t.astype({c:np.float32 for c in t.columns})\n        t.reset_index(inplace=True)\n        df1=df1.merge(t,on=x,how='left')\n        gc.collect()\n    return df1\nprint(len(df))\ndf=agg_functions(df)\nprint(len(df))\ntrain=agg_functions(train)\n\n# coverfull worn on data by train data mean\ns=train.deviceIdstd_bidWin.mean()\ndf['deviceIdstd_bidWin'].fillna(s,inplace=True)\ns=train.deviceIdmin_bidWin.mean()\ndf['deviceIdmin_bidWin'].fillna(s,inplace=True)\ns=train.deviceIdmax_bidWin.mean()\ndf['deviceIdmax_bidWin'].fillna(s,inplace=True)\ns=train.deviceIdmean_bidWin.mean()\ndf['deviceIdmean_bidWin'].fillna(s,inplace=True)\ns=train.deviceIdmedian_bidWin.mean()\ndf['deviceIdmedian_bidWin'].fillna(s,inplace=True)\ns=train.deviceIdvar_bidWin.mean()\ndf['deviceIdvar_bidWin'].fillna(s,inplace=True)\ns=train.deviceIdsem_bidWin.mean()\ndf['deviceIdsem_bidWin'].fillna(s,inplace=True)\ndel train\ngc.collect()\n# drop feature not present in training session\ncols=['unitDisplayTypemin_bidWin']\ndf.drop(cols,axis=1,inplace=True)\ndisplay(df.isnull().sum())","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-21T12:21:05.767839Z","iopub.execute_input":"2022-09-21T12:21:05.768347Z","iopub.status.idle":"2022-09-21T12:22:19.526416Z","shell.execute_reply.started":"2022-09-21T12:21:05.768248Z","shell.execute_reply":"2022-09-21T12:22:19.525319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing data part","metadata":{}},{"cell_type":"code","source":"\n\n# PREPARE FEATURES\ndf['size']=df['size'].apply(lambda x: x.split('x'))\ndf['size']=df['size'].apply(lambda x: int(x[0])*int(x[1]) )\ndf['size']=df['size']*0.0001\n\ndf.mediationProviderVersion=df.mediationProviderVersion.apply(lambda x: x.replace('-beta','.') if 'beta' in x else x+'.0' )\ndef ver_convert_num(ver):\n    ver=ver.str.split('.',expand=True)\n    digit=[2, 1, 1, 2]\n    for i,x in enumerate(ver.columns):\n        if int(ver[x].str.len().min())==digit[i]:\n            continue\n        ver[x]=ver[x].apply(lambda x: x if len(x)==digit[i] else (digit[i]-len(x))*'0'+x )\n    ver=ver.apply(lambda x: int(''.join(x)),axis=1)\n    return ver\ndf.mediationProviderVersion=ver_convert_num(df.mediationProviderVersion)\nrank={102100: 0, 103100: 1, 103200: 2, 103400: 3, 103500: 4, 103600: 5, 103700: 6, 110000: 7, 110003: 8, 110004: 9, 110005: 10,\n      110007: 11, 110101: 12, 110106: 13, 110110: 14, 111000: 15, 111100: 16, 111102: 17, 111103: 18, 111104: 19, 111200: 20, 111305: 21,\n      112003: 22, 112100: 23,113100: 24, 113200: 25, 113201: 26, 113300: 27, 114000: 28, 114100: 29, 114200: 30, 114201: 31, 114300: 32,\n      114304: 33, 114400: 34}\nunranked=[]\nfor x in df.mediationProviderVersion.unique():\n    if x not in rank.keys():\n        unranked.append(x)\nt=list(rank.keys())\ntem=[i  for i in unranked if i<t[0]]\ndef interver(tem,border,index):\n    global rank\n    tem.sort()\n    if len(tem)>0:\n        tem.append(border)\n        for i,c in enumerate(tem):\n            if c==border:\n                continue\n            rank[c]=index-((i+1)/len(tem))\ninterver(tem,t[0],0)\nfor x in range(0,len(t),2):\n    if x+1==len(t):\n        continue\n    tem=[i for i in unranked if i<t[x+1] and i>t[x] ]\n    interver(tem,t[x+1],x+1)\n    tem=[i for i in unranked if i<t[x] and i>t[x-1] ]\n    interver(tem,t[x],x)\ntem=[i  for i in unranked if i>t[-1]]\ntem.sort()\nif len(tem)>0:\n    h=rank[t[-1]]+1\n    for i,c in enumerate(tem):\n        if i+1==len(tem):\n            rank[c]=h\n            continue\n        rank[c]=h-((i+1)/len(tem))\nprint(rank)\ndf.mediationProviderVersion=df.mediationProviderVersion.apply(lambda x: rank[x])\n\ndf['os']=df.osAndVersion.apply(lambda x: x.split('-')[0])\ndf['ver']=df.osAndVersion.apply(lambda x: x.split('-')[1].split('.')[0]) \nos_rank=[{4: 0, 5: 1, 6: 2, 7: 3, 8: 4, 9: 5, 10: 6, 11: 7},\n{10: 0, 11: 1, 12: 2, 13: 3, 14: 4, 15: 5, 16: 6}]\nthreshold=[3,1]\nfor i,c in enumerate(df.os.unique()):\n    rank=os_rank[i]\n    for x in df.loc[df.os==c,'ver'].unique():\n        if int(x) not in rank.keys():\n            print(x)\n            g=rank.keys()\n            x=int(x)\n            rank[x]=min(g)-x if x<min(g) else rank[max(g)]+(x-max(g))\n    \n    print(rank)\n    df.loc[df.os==c,'ver']=df.loc[df.os==c,'ver'].apply(lambda y:rank[int(y)]+threshold[i])\ndel df['os']\ndf['ver']=df['ver'].astype(np.float32)\n\nimport gc\ngc.collect()\ndf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-21T12:22:19.529124Z","iopub.execute_input":"2022-09-21T12:22:19.529584Z","iopub.status.idle":"2022-09-21T12:22:20.510813Z","shell.execute_reply.started":"2022-09-21T12:22:19.529517Z","shell.execute_reply":"2022-09-21T12:22:20.509440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Countrycode to Human Development Index","metadata":{}},{"cell_type":"code","source":"import pycountry\ncountries = {}\nfor country in pycountry.countries:\n    countries[country.name] = country.alpha_2\n    \ncountry=pd.read_csv('../input/country-name-hdi/csvData.csv')\ns=country.hdi2019.median()\ncountry['countryCode']=country.country.apply(lambda x: countries.get(x, '0_0'))\ncountry.loc[country.countryCode=='0_0','hdi2019']=s\ncountry.drop(['pop2022','country'],axis=1,inplace=True)\ncountry.drop_duplicates(subset='countryCode',keep='first',inplace=True)\ncountry.set_index('countryCode',inplace=True)\ncountry=country.loc[:,'hdi2019'].to_dict()\ndf.countryCode.fillna('0_0',inplace=True)\ndf['countryCode']=df['countryCode'].apply(lambda x: country.get(x,s))\ndf['countryCode']=df['countryCode'].astype(np.float32)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-09-21T12:22:20.512554Z","iopub.execute_input":"2022-09-21T12:22:20.513042Z","iopub.status.idle":"2022-09-21T12:22:20.621864Z","shell.execute_reply.started":"2022-09-21T12:22:20.512999Z","shell.execute_reply":"2022-09-21T12:22:20.620932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Categories to one-hot Feature","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\ndef preprocess(df1):\n    unique_dict={'unitDisplayType' :['rewarded', 'interstitial', 'banner'],\n        'bundleId' :['com.tilegarden.match3', 'com.loop.match3d','com.tintash.nailsalon', '1502447854', 'se.ace.fishinc','com.kamilbilge.ropesavior3d', '1523081624','com.AppIdeas.LevelUpRunner', '1529614832', '1586795332','dovi.coalmining.inc', '1542256628', '1436213906','com.YayySAL.DodgeAgent', 'com.volt.dresstoimpress', '1579489488','1582745578', '1569586264'],\n        'connectionType' :['3G', 'WIFI', 'UNKNOWN', '0_0'],\n        'c1' :['cb2', '7d3', '8bd', 'ad3', '59b', 'fdc', '7ca', '7b8', '9b5','dfa', '1ba', '2c1', 'ce2', '3db', '76e', '313', 'f8d', 'c2e','82a', 'b22', 'f0f', '3dc', 'c17', '403', '707', '94f', 'cb3','87b', '7be', 'd69', '00e', 'ea0', '397', '443', '466', '16a','1c3', 'f99', 'f17', '05b', '2f6', '8f8', '49e', 'a65', '064','da9', 'd79', 'ab5', 'fd5', '541'],\n        'c3' :['6b', '79', '4b', '4e'],\n        'c2':['6.0', '3.0', '9.0', '7.0', '8.0', '5.0', '2.0', '1.0','4.0']\n        }\n    def unique_one_hot(df2,name):\n        df2[name]=df2[name].astype(str)\n        df2[name].fillna('0_0',inplace=True)\n        df2[name]=df2[name].str.replace('nan','0_0')\n        unique=unique_dict[name]\n        dummies={ x:y for x,y in zip(unique,pd.get_dummies(unique,dtype=np.uint8).values)}\n        one_hot=pd.DataFrame(columns=[f'{name}_{i}' for i,x in enumerate(unique)])\n        print(name)\n        one_hot[[f'{name}_{i}' for i,x in enumerate(unique)]]=np.array([dummies[x]  for x in df2[name]])\n        one_hot=one_hot.astype({f'{name}_{i}':np.uint8 for i,x in enumerate(unique)})\n        df2=df2.join(one_hot)\n        return df2\n    categorical=['unitDisplayType','connectionType','bundleId','c3','c1']\n    for cat in categorical:\n        df1=unique_one_hot(df1,cat)\n    \n    df1.drop(categorical,axis=1,inplace=True)\n    gc.collect()\n    return df1\ndf=preprocess(df)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T12:22:20.624436Z","iopub.execute_input":"2022-09-21T12:22:20.625168Z","iopub.status.idle":"2022-09-21T12:22:21.599405Z","shell.execute_reply.started":"2022-09-21T12:22:20.625123Z","shell.execute_reply":"2022-09-21T12:22:21.598187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Display Processed data ","metadata":{}},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-09-21T12:22:21.600955Z","iopub.execute_input":"2022-09-21T12:22:21.601384Z","iopub.status.idle":"2022-09-21T12:22:21.648066Z","shell.execute_reply.started":"2022-09-21T12:22:21.601288Z","shell.execute_reply":"2022-09-21T12:22:21.646953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and save values for submission","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf \ndc=['eventTimestamp','brandName','appVersion','correctModelName','deviceId','osAndVersion']\nmodel=tf.keras.models.load_model('../input/eda-item-price/train_76.408043.h5')\nout=model.predict(df.drop(dc,axis=1),batch_size=512)\ndf=df.loc[:,['deviceId']]\ndf['winBid']=out\ndf.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T12:22:21.649431Z","iopub.execute_input":"2022-09-21T12:22:21.649792Z","iopub.status.idle":"2022-09-21T12:22:28.851979Z","shell.execute_reply.started":"2022-09-21T12:22:21.649761Z","shell.execute_reply":"2022-09-21T12:22:28.848986Z"},"trusted":true},"execution_count":null,"outputs":[]}]}