{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This NoteBook Prepare train data for training phase\n### if I make any mistake or inefficient operation to preprocess phase then please comments it","metadata":{}},{"cell_type":"markdown","source":"## Import Necessary files and Modules","metadata":{}},{"cell_type":"code","source":"%%capture\n!pip install swifter\n\nimport numpy as np # linear algebra\nimport pandas as pd\nimport swifter\nimport gc\nfrom tqdm import tqdm # data processing, CSV file I/O (e.g. pd.read_csv)\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n\ndf = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/train_data.csv')\ndf.connectionType.fillna('0_0',inplace=True)\n# test  = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/test_data.csv')\ndisplay(df.isnull().sum())\ndisplay(df.nunique())\ndisplay()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-21T04:20:02.824951Z","iopub.execute_input":"2022-09-21T04:20:02.825477Z","iopub.status.idle":"2022-09-21T04:21:13.946166Z","shell.execute_reply.started":"2022-09-21T04:20:02.825374Z","shell.execute_reply":"2022-09-21T04:21:13.944422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Actually Preprocessing Phase ","metadata":{}},{"cell_type":"code","source":"def agg_functions(df1):\n    features=['c1','deviceId','unitDisplayType','bundleId']\n    for x in tqdm(features):\n        t=df1.groupby(x)['winBid'].agg(['std','max','min','mean','median','var','sem'])\n        t.columns=[x+c+'_bidWin' for c in t.columns]\n        t=t.astype({c:np.float32 for c in t.columns})\n        t.reset_index(inplace=True)\n        df1=df1.merge(t,on=x)\n        gc.collect()\n    return df1\n# AGGREGATE SPECIFIC FEATURES WITH WINBID\ndf=agg_functions(df)\n\n\ndef drop_features(df1):\n    cols=[]\n    for x in df1.columns:\n        if df1[x].nunique()==1:\n            cols.append(x)\n    print(len(cols),cols)\n    return df1.drop(cols,axis=1)\n# DROP FEATURES WITH SINGE UNIQUE VALUE\ndf=drop_features(df)\n\n# SIZE COLUMNS CONVERT INTO SURFACE AREA \ndf['size']=df['size'].apply(lambda x: x.split('x'))\ndf['size']=df['size'].apply(lambda x: int(x[0])*int(x[1]) )\ndf['size']=df['size']*0.0001\ndf['size']=df['size'].astype(np.float32)\ngc.collect()\n\n# MEDIATION VERSIONS CONVERT INTO RANK FEATURE\ndf.mediationProviderVersion=df.mediationProviderVersion.swifter.apply(lambda x: x.replace('-beta','.') if 'beta' in x else x+'.0' )\n\ndef ver_convert_num(ver):\n    ver=ver.str.split('.',expand=True)\n    digit=[int(ver[x].str.len().max()) for x in ver.columns]\n    print(digit)\n    for i,x in enumerate(ver.columns):\n        if int(ver[x].str.len().min())==digit[i]:\n            continue\n        ver[x]=ver[x].swifter.apply(lambda x: x if len(x)==digit[i] else (digit[i]-len(x))*'0'+x )\n    ver=ver.swifter.apply(lambda x: int(''.join(x)),axis=1)\n    return ver\ndf.mediationProviderVersion=ver_convert_num(df.mediationProviderVersion)\ngc.collect()\nrank={y:x for x,y in enumerate(np.sort(df.mediationProviderVersion.unique()))}\nprint(rank)\ndf.mediationProviderVersion=df.mediationProviderVersion.swifter.apply(lambda x: rank[x])\n\n# VERSION OF OS CONVERT TO RANK WITH SPECIFIC OPERATING SYSTEM\ndf['os']=df.osAndVersion.apply(lambda x: x.split('-')[0])\ndf['ver']=df.osAndVersion.apply(lambda x: x.split('-')[1].split('.')[0])\ngc.collect()\nthreshold=[3,1]\nfor i,c in enumerate(df.os.unique()):\n    rank={y:x for x,y in enumerate(np.sort(df.loc[df.os==c,'ver'].astype(np.int8).unique()))}\n    print(rank)\n    df.loc[df.os==c,'ver']=df.loc[df.os==c,'ver'].apply(lambda y:rank[int(y)]+threshold[i])\ndel df['os']\ndf['ver']=df['ver'].astype(np.float32)\ngc.collect()\n\n\n# DROPOUT FEATURES WHICH MODEL CAN'T TRAIN ON THEM \ncols=['eventTimestamp','brandName','appVersion','correctModelName','deviceId','osAndVersion','has_won']\n\ndf.drop(cols,axis=1,inplace=True)\ngc.collect()\n\n# COUNTRYCODE COLUMNS REPLACE HIS COUNTRY HUMAN DEVELOPMENT INDEX\nimport pycountry\ncountries = {}\nfor country in pycountry.countries:\n    countries[country.name] = country.alpha_2\n    \ncountry=pd.read_csv('../input/country-name-hdi/csvData.csv')\ns=country.hdi2019.median()\ncountry['countryCode']=country.country.apply(lambda x: countries.get(x, '0_0'))\ncountry.loc[country.countryCode=='0_0','hdi2019']=s\ncountry.drop(['pop2022','country'],axis=1,inplace=True)\ncountry.drop_duplicates(subset='countryCode',keep='first',inplace=True)\ncountry.set_index('countryCode',inplace=True)\ncountry=country.loc[:,'hdi2019'].to_dict()\ndf.countryCode.fillna('0_0',inplace=True)\ndf['countryCode']=df['countryCode'].apply(lambda x: country.get(x,s))\ndf['countryCode']=df['countryCode'].astype(np.float32)\n\n# THIS PART WE CONVERTING CATEGORIES TO ONE-HOT VALUES\nfrom tqdm import tqdm\ndef one_hot_conversion(df1):\n    unique_dict={'unitDisplayType' :['rewarded', 'interstitial', 'banner'],\n        'bundleId' :['com.tilegarden.match3', 'com.loop.match3d','com.tintash.nailsalon', '1502447854', 'se.ace.fishinc','com.kamilbilge.ropesavior3d', '1523081624','com.AppIdeas.LevelUpRunner', '1529614832', '1586795332','dovi.coalmining.inc', '1542256628', '1436213906','com.YayySAL.DodgeAgent', 'com.volt.dresstoimpress', '1579489488','1582745578', '1569586264'],\n        'connectionType' :['3G', 'WIFI', 'UNKNOWN', '0_0'],\n        'c1' :['cb2', '7d3', '8bd', 'ad3', '59b', 'fdc', '7ca', '7b8', '9b5','dfa', '1ba', '2c1', 'ce2', '3db', '76e', '313', 'f8d', 'c2e','82a', 'b22', 'f0f', '3dc', 'c17', '403', '707', '94f', 'cb3','87b', '7be', 'd69', '00e', 'ea0', '397', '443', '466', '16a','1c3', 'f99', 'f17', '05b', '2f6', '8f8', '49e', 'a65', '064','da9', 'd79', 'ab5', 'fd5', '541'],\n        'c3' :['6b', '79', '4b', '4e'],\n        'c2':['6.0', '3.0', '9.0', '7.0', '8.0', '5.0', '2.0', '1.0','4.0']\n        }\n    def unique_one_hot(df2,name):\n        df2[name]=df2[name].astype(str)\n        df2[name].fillna('0_0',inplace=True)\n        df2[name]=df2[name].str.replace('nan','0_0')\n        unique=unique_dict[name]\n        dummies={ x:y for x,y in zip(unique,pd.get_dummies(unique,dtype=np.uint8).values)}\n        print('j')\n        one_hot=pd.DataFrame(columns=[f'{name}_{i}' for i,x in enumerate(unique)])\n        print(name)\n        one_hot[[f'{name}_{i}' for i,x in enumerate(unique)]]=np.array([dummies[x]  for x in df2[name]])\n        one_hot=one_hot.astype({f'{name}_{i}':np.uint8 for i,x in enumerate(unique)})\n        df2=df2.join(one_hot)\n        return df2\n    categorical=['unitDisplayType','connectionType','bundleId','c3','c1']\n\n    for cat in categorical:\n        df1=unique_one_hot(df1,cat)\n    df1.drop(categorical,axis=1,inplace=True)\n    gc.collect()\n    return df1\ndf=one_hot_conversion(df)\n\n# FINISHING TOUCH PART TO ASSIGNING LOW PRECISION TYPES FOR LARGE MEMORY CONSUMING FEATURES\ncols=[np.float32 if df[x].dtype==np.float64 else df[x].dtype for x in df.columns]\ncols={x:y for x,y in zip(df.columns,cols)}\ndf=df.astype(cols)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-21T04:21:13.949458Z","iopub.execute_input":"2022-09-21T04:21:13.950008Z","iopub.status.idle":"2022-09-21T04:30:32.802741Z","shell.execute_reply.started":"2022-09-21T04:21:13.949958Z","shell.execute_reply":"2022-09-21T04:30:32.801741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Finally Save Part","metadata":{}},{"cell_type":"code","source":"print(df.isnull().sum().sum())\nprint(df.columns)\ndf.to_csv('prepared_train.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T04:30:32.804676Z","iopub.execute_input":"2022-09-21T04:30:32.805876Z","iopub.status.idle":"2022-09-21T04:35:46.208233Z","shell.execute_reply.started":"2022-09-21T04:30:32.805828Z","shell.execute_reply":"2022-09-21T04:35:46.206743Z"},"trusted":true},"execution_count":null,"outputs":[]}]}