{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# DTABP 1st Place Solution\nThis notebook consists three major parts that can be run separately:\n* EDA\n* Transform\n* Model\n\nYou can notice many commented lines, which I decided to leave as is. Abbreviations (both in comments and code):\n* eTs, ets = eventTimeStamp\n* dId, did = deviceId\n* uDT, udt = unitDisplayType\n\nThank you for such an exciting competition!\n# EDA","metadata":{}},{"cell_type":"markdown","source":"## Import packages\n","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom datetime import datetime\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:12:28.104948Z","iopub.execute_input":"2022-10-08T20:12:28.105853Z","iopub.status.idle":"2022-10-08T20:12:28.133823Z","shell.execute_reply.started":"2022-10-08T20:12:28.105748Z","shell.execute_reply":"2022-10-08T20:12:28.132591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading and first look at the data","metadata":{}},{"cell_type":"code","source":"%%time\nidir = '../input/digital-turbine-auction-bid-price-prediction/'\ntrain = pd.read_csv(f'{idir}/train_data.csv')\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:12:28.138752Z","iopub.execute_input":"2022-10-08T20:12:28.139426Z","iopub.status.idle":"2022-10-08T20:13:07.540455Z","shell.execute_reply.started":"2022-10-08T20:12:28.139388Z","shell.execute_reply":"2022-10-08T20:13:07.539324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(f'{idir}/test_data.csv')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:13:07.542247Z","iopub.execute_input":"2022-10-08T20:13:07.542621Z","iopub.status.idle":"2022-10-08T20:13:07.792759Z","shell.execute_reply.started":"2022-10-08T20:13:07.542591Z","shell.execute_reply":"2022-10-08T20:13:07.791514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Timestamp preprocess\nI started from looking at the timestamp = processing it into year, month, hour.\nProbably time of a day would be valuable but this will require timezones per countries. Function presented below is superslow :(","metadata":{}},{"cell_type":"code","source":"def ts_conv(ts):\n    t = datetime.utcfromtimestamp(int(ts/1000))\n    # year = t.strftime('%Y') all data is from 2022\n    month = t.strftime('%m') # all data is 06-07\n    day = t.strftime('%d')\n    hour = t.strftime('%H')\n    return month, day, hour\ndef preprocess(d):\n    d['month'], d['day'], d['hour'] = zip(*d.apply(lambda x: ts_conv(x.eventTimestamp), axis=1))\n    return d","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:13:07.794507Z","iopub.execute_input":"2022-10-08T20:13:07.795003Z","iopub.status.idle":"2022-10-08T20:13:07.803565Z","shell.execute_reply.started":"2022-10-08T20:13:07.794960Z","shell.execute_reply":"2022-10-08T20:13:07.802293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = preprocess(train)\ntest = preprocess(test)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:13:07.806569Z","iopub.execute_input":"2022-10-08T20:13:07.807030Z","iopub.status.idle":"2022-10-08T20:16:29.632896Z","shell.execute_reply.started":"2022-10-08T20:13:07.806988Z","shell.execute_reply":"2022-10-08T20:16:29.631643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Value counts on categorical data\nThere is a lot of differences between train and test data, the reason is explained later.\nDifferences in: \n\n* bundleId\n* utd\n* appVersion\n* countryCode\n* c1\n* size","metadata":{}},{"cell_type":"code","source":"def vc(col):\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))\n    train_sc = train[col].value_counts().sort_index(ascending=False).rename(f'Train {col} per device')\n    ax1.pie(train_sc, labels=train_sc.index)\n    ax1.set_title(train_sc.name)\n    test_sc = test[col].value_counts().sort_index(ascending=False).rename(f'Test {col} per device')\n    ax2.pie(test_sc, labels=test_sc.index)\n    ax2.set_title(test_sc.name)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:16:29.634644Z","iopub.execute_input":"2022-10-08T20:16:29.635672Z","iopub.status.idle":"2022-10-08T20:16:29.643537Z","shell.execute_reply.started":"2022-10-08T20:16:29.635634Z","shell.execute_reply":"2022-10-08T20:16:29.642316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor fi in test.columns:\n    if test[fi].dtype in [np.int64, np.float64]:\n        print(f'numeric variable {fi}')\n    elif fi == 'deviceId':\n        print(f'special object variable {fi}')\n    else:\n        print(f'object variable {fi}')\n        vc(fi)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:16:29.645149Z","iopub.execute_input":"2022-10-08T20:16:29.645521Z","iopub.status.idle":"2022-10-08T20:17:32.171911Z","shell.execute_reply.started":"2022-10-08T20:16:29.645489Z","shell.execute_reply":"2022-10-08T20:17:32.170806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA part2\nIt turns out that in training data there are a lot of auctions per dId compared to test data - thats probably one of reasons of distribution differences presented above:","metadata":{}},{"cell_type":"code","source":"%%time\ntrain.groupby('deviceId')['winBid'].count()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:17:32.173452Z","iopub.execute_input":"2022-10-08T20:17:32.173796Z","iopub.status.idle":"2022-10-08T20:17:33.456956Z","shell.execute_reply.started":"2022-10-08T20:17:32.173765Z","shell.execute_reply":"2022-10-08T20:17:33.456035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby('deviceId')['brandName'].count().reset_index().sort_values(by = 'brandName')","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:17:33.458237Z","iopub.execute_input":"2022-10-08T20:17:33.458820Z","iopub.status.idle":"2022-10-08T20:17:33.529002Z","shell.execute_reply.started":"2022-10-08T20:17:33.458784Z","shell.execute_reply":"2022-10-08T20:17:33.527753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All of training dId can be found in test data:","metadata":{}},{"cell_type":"code","source":"train.merge(test, on = 'deviceId').shape","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:17:33.530777Z","iopub.execute_input":"2022-10-08T20:17:33.531138Z","iopub.status.idle":"2022-10-08T20:18:33.662017Z","shell.execute_reply.started":"2022-10-08T20:17:33.531105Z","shell.execute_reply":"2022-10-08T20:18:33.660830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Going further with relationship beetween with ets and did:","metadata":{}},{"cell_type":"code","source":"cnt = train.groupby('deviceId')['winBid'].count()\ncnt = cnt.reset_index()\ncnt.columns = ['deviceId','cnt']\ncnt","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:33.666033Z","iopub.execute_input":"2022-10-08T20:18:33.666442Z","iopub.status.idle":"2022-10-08T20:18:34.924082Z","shell.execute_reply.started":"2022-10-08T20:18:33.666402Z","shell.execute_reply":"2022-10-08T20:18:34.922768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's take an example did - there are over 4k rows in training data per this device:","metadata":{}},{"cell_type":"code","source":"d0 = train[train['deviceId'].isin(['80fffd4d742'])].copy()\neTs = 'eventTimestamp'\nd0.sort_values(by = eTs)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:34.925970Z","iopub.execute_input":"2022-10-08T20:18:34.926353Z","iopub.status.idle":"2022-10-08T20:18:39.915042Z","shell.execute_reply.started":"2022-10-08T20:18:34.926319Z","shell.execute_reply":"2022-10-08T20:18:39.913762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"... and only one row in test device. Also this is the last auction (maximum ets):","metadata":{}},{"cell_type":"code","source":"test[test['deviceId'].isin(['80fffd4d742'])]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:39.916727Z","iopub.execute_input":"2022-10-08T20:18:39.917543Z","iopub.status.idle":"2022-10-08T20:18:39.958543Z","shell.execute_reply.started":"2022-10-08T20:18:39.917493Z","shell.execute_reply":"2022-10-08T20:18:39.957511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Number of devices in train:","metadata":{}},{"cell_type":"code","source":"len(train['deviceId'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:39.960036Z","iopub.execute_input":"2022-10-08T20:18:39.960382Z","iopub.status.idle":"2022-10-08T20:18:40.814049Z","shell.execute_reply.started":"2022-10-08T20:18:39.960345Z","shell.execute_reply":"2022-10-08T20:18:40.812747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Number of devices in test:","metadata":{}},{"cell_type":"code","source":"len(test['deviceId'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:40.815661Z","iopub.execute_input":"2022-10-08T20:18:40.816012Z","iopub.status.idle":"2022-10-08T20:18:40.830459Z","shell.execute_reply.started":"2022-10-08T20:18:40.815983Z","shell.execute_reply":"2022-10-08T20:18:40.828802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Devices with 2 auctions:","metadata":{}},{"cell_type":"code","source":"cnt[cnt['cnt'].isin([2])]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:40.831853Z","iopub.execute_input":"2022-10-08T20:18:40.832454Z","iopub.status.idle":"2022-10-08T20:18:40.849541Z","shell.execute_reply.started":"2022-10-08T20:18:40.832415Z","shell.execute_reply":"2022-10-08T20:18:40.848684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking if rule holds for every device: last deviceId auction in test set.\nMinimum difference between timestamps is 2, so that is true.","metadata":{}},{"cell_type":"code","source":"eTs = 'eventTimestamp'\ndId = 'deviceId'\nt0 = train.groupby(dId).last()\n\nd = test.merge(t0, on = 'deviceId')[[eTs+'_x', eTs + '_y']]\nd['diff'] = d[eTs+'_x'] - d[eTs + '_y']\nmin(d['diff'])","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:40.850764Z","iopub.execute_input":"2022-10-08T20:18:40.851461Z","iopub.status.idle":"2022-10-08T20:18:53.390529Z","shell.execute_reply.started":"2022-10-08T20:18:40.851425Z","shell.execute_reply":"2022-10-08T20:18:53.389271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"... and maximum is huge! So these kind of auctions could actually be not related to each other","metadata":{}},{"cell_type":"code","source":"max(d['diff'])","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:53.392198Z","iopub.execute_input":"2022-10-08T20:18:53.392683Z","iopub.status.idle":"2022-10-08T20:18:53.405010Z","shell.execute_reply.started":"2022-10-08T20:18:53.392639Z","shell.execute_reply":"2022-10-08T20:18:53.403632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Numerical data\nSo far I didn't have time to explore numerical data, besides:\n* eTs\n* histograms of c2 and c4 looks suspicious\n* notice that some of public notebooks treats c2 and c4 as categorical variables\n* size can be treated as two (hxw) numerical variables, maybe three (size = h * w )\n* target winBid has a lot of extremes values, I tried np.log(winBid) but score is worse\n* target winBid should be >= 0.01","metadata":{}},{"cell_type":"code","source":"train['c4'].hist()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:53.406429Z","iopub.execute_input":"2022-10-08T20:18:53.406800Z","iopub.status.idle":"2022-10-08T20:18:53.943549Z","shell.execute_reply.started":"2022-10-08T20:18:53.406756Z","shell.execute_reply":"2022-10-08T20:18:53.942276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['c2'].hist()#.map(lambda x: np.log(x)).hist(bins = 200)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:53.945465Z","iopub.execute_input":"2022-10-08T20:18:53.946540Z","iopub.status.idle":"2022-10-08T20:18:54.469162Z","shell.execute_reply.started":"2022-10-08T20:18:53.946487Z","shell.execute_reply":"2022-10-08T20:18:54.468325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clean up","metadata":{}},{"cell_type":"code","source":"import gc\ndel test,train,d,t0,cnt\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:54.471502Z","iopub.execute_input":"2022-10-08T20:18:54.472713Z","iopub.status.idle":"2022-10-08T20:18:55.846188Z","shell.execute_reply.started":"2022-10-08T20:18:54.472660Z","shell.execute_reply":"2022-10-08T20:18:55.845046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data\nGiven informations presented below, I started transforming and splitting data. Steps in transform are:\n* concat train and test into one dataframe (d)\n* create some features, i.e shiftWinBids, utdShiftWinBid, etc\n* split data again and again\n\n## Import packages","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom datetime import datetime\nimport numpy as np\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import train_test_split\nimport gc","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-10-08T20:18:55.847616Z","iopub.execute_input":"2022-10-08T20:18:55.847934Z","iopub.status.idle":"2022-10-08T20:18:56.698573Z","shell.execute_reply.started":"2022-10-08T20:18:55.847907Z","shell.execute_reply":"2022-10-08T20:18:56.697180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some parameters, comments for metadata holded in dict P","metadata":{}},{"cell_type":"code","source":"P = {}\nP['DATAVERSION'] = 'v89'\nP['DATACOMMENT'] = 'back to standard size group'\nP['TOPN'] = None\nP['NFOLDS'] = 5\nP['GROUP'] = 'size'","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:56.699987Z","iopub.execute_input":"2022-10-08T20:18:56.700413Z","iopub.status.idle":"2022-10-08T20:18:56.707574Z","shell.execute_reply.started":"2022-10-08T20:18:56.700375Z","shell.execute_reply":"2022-10-08T20:18:56.706348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train[ets].min() # 1656290100421","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:56.709006Z","iopub.execute_input":"2022-10-08T20:18:56.709454Z","iopub.status.idle":"2022-10-08T20:18:56.718401Z","shell.execute_reply.started":"2022-10-08T20:18:56.709419Z","shell.execute_reply":"2022-10-08T20:18:56.717024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nidir = '../input/digital-turbine-auction-bid-price-prediction/'\ntrain = pd.read_csv(f'{idir}/train_data.csv', nrows = P['TOPN'])\nets = 'eventTimestamp'\ndid = 'deviceId'\nudt = 'unitDisplayType'\n#train.sort_values(by = ets, inplace = True)\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:18:56.719961Z","iopub.execute_input":"2022-10-08T20:18:56.720701Z","iopub.status.idle":"2022-10-08T20:19:23.397917Z","shell.execute_reply.started":"2022-10-08T20:18:56.720656Z","shell.execute_reply":"2022-10-08T20:19:23.396620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest = pd.read_csv(f'{idir}/test_data.csv', nrows = P['TOPN'])\n#test.sort_values(by = 'deviceId',inplace = True)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:19:23.399341Z","iopub.execute_input":"2022-10-08T20:19:23.399800Z","iopub.status.idle":"2022-10-08T20:19:23.532664Z","shell.execute_reply.started":"2022-10-08T20:19:23.399753Z","shell.execute_reply":"2022-10-08T20:19:23.531355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are only few examples with 480x320 and 1024x768 so I renamed them into popular one:","metadata":{}},{"cell_type":"code","source":"d = pd.concat([train,test])\n#d['size_udt'] = d['size']+'_' + d[udt]\nd.loc[d['size'] == '480x320','size'] = '320x480'\nd.loc[d['size'] == '1024x768','size'] = '768x1024'\ndel train, test","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:19:23.534236Z","iopub.execute_input":"2022-10-08T20:19:23.534608Z","iopub.status.idle":"2022-10-08T20:19:29.017630Z","shell.execute_reply.started":"2022-10-08T20:19:23.534575Z","shell.execute_reply":"2022-10-08T20:19:29.016355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sorting d by did and ets for shifts, and creating grouped dataFrame by 'size' (called dg). At some point I tried this idea of grouping and it helped a lot, but I'm still not 100% sure about this.","metadata":{}},{"cell_type":"code","source":"%%time\n#p=4/5\n#d['winBid'] = d['winBid']**p\nd.sort_values(by = [did,ets], inplace = True)\nprint(d.shape)\ndg = d.groupby([did,P['GROUP']], as_index = False)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:19:29.019554Z","iopub.execute_input":"2022-10-08T20:19:29.020242Z","iopub.status.idle":"2022-10-08T20:19:56.605089Z","shell.execute_reply.started":"2022-10-08T20:19:29.020170Z","shell.execute_reply":"2022-10-08T20:19:56.603769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d[did].unique()[1]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:19:56.614924Z","iopub.execute_input":"2022-10-08T20:19:56.615365Z","iopub.status.idle":"2022-10-08T20:19:57.948179Z","shell.execute_reply.started":"2022-10-08T20:19:56.615319Z","shell.execute_reply":"2022-10-08T20:19:57.946860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"did0 = '6a0019a885a'\nd[d[did].isin([did0]) & (d[udt].isin(['interstitial','rewarded']))]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:19:57.949621Z","iopub.execute_input":"2022-10-08T20:19:57.949999Z","iopub.status.idle":"2022-10-08T20:20:02.524108Z","shell.execute_reply.started":"2022-10-08T20:19:57.949964Z","shell.execute_reply":"2022-10-08T20:20:02.523025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Shifting winBid gives an improvement at the beginning, so this features holds last 10 winBids for each (did x ets). NaNs if there are no previous auctions","metadata":{}},{"cell_type":"code","source":"%%time\nshift_win_bids = []\nfor fi in range(1,10):\n    col = f'shift{fi}WinBid'\n    d[col] = dg['winBid'].shift(fi)\n    shift_win_bids.append(col)\n    #d[f'shift{fi}C4'] = dg['c4'].shift(fi)\n    #d[f'shift{fi}SentPrice'] = dg['sentPrice'].shift(fi)\n    #d[f'shift{fi}UnitDisplayType'] = dg['unitDisplayType'].shift(fi)\n    #d[f'shift{fi}Size'] = '' #dg['size'].shift(fi)\nprint(d.shape)\ndel dg\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:20:02.525861Z","iopub.execute_input":"2022-10-08T20:20:02.527192Z","iopub.status.idle":"2022-10-08T20:20:08.318517Z","shell.execute_reply.started":"2022-10-08T20:20:02.527144Z","shell.execute_reply":"2022-10-08T20:20:08.317324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## udtShift1WinBid\nThis is my favourite feature (so far).\nSo the idea behind this, is to given auction (did x ets) join last winBid per uDT).\n\nIf at time t0, uDT=rewarded, winBid = 10, at t1, t2, t3 and t4 auctions were not rewarded, then t5 is rewarded again then udtShift1WinBid = 10.\n\nEngineering: \n* pivot\n* shift\n* fill.na(method = 'ffill')\n* gather into one variable (probably not necessary)\n\nPS, I've probably found a bug in pandas while creating it\n\nExample presented below:\n\n","metadata":{}},{"cell_type":"code","source":"did0 = d[did].unique()[1]\nd[d[did].isin([did0]) & (d[udt].isin(['interstitial','rewarded']))]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:20:08.320104Z","iopub.execute_input":"2022-10-08T20:20:08.320512Z","iopub.status.idle":"2022-10-08T20:20:11.042600Z","shell.execute_reply.started":"2022-10-08T20:20:08.320477Z","shell.execute_reply":"2022-10-08T20:20:11.041276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Meanwhile I've found (before pivoting data) duplicates (did x ets) but I didn't have time so far to investigate it further.","metadata":{}},{"cell_type":"code","source":"%%time\nd.drop_duplicates(subset = [did,ets],inplace = True)\nd.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:20:11.044232Z","iopub.execute_input":"2022-10-08T20:20:11.045671Z","iopub.status.idle":"2022-10-08T20:20:18.661354Z","shell.execute_reply.started":"2022-10-08T20:20:11.045623Z","shell.execute_reply":"2022-10-08T20:20:18.660090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nshift_udt_cls = []\nd00 = d[[did,ets,'winBid',udt]].pivot([did,ets],columns = udt, values = 'winBid').reset_index()#.fillna(method='ffill').reset_index()\nd00.sort_values(by = [did,ets],inplace = True)\nd00g = d00.groupby(did)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:20:18.662959Z","iopub.execute_input":"2022-10-08T20:20:18.663445Z","iopub.status.idle":"2022-10-08T20:20:43.670828Z","shell.execute_reply.started":"2022-10-08T20:20:18.663401Z","shell.execute_reply":"2022-10-08T20:20:43.669424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d00[d00[did].isin([did0])]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:20:43.672514Z","iopub.execute_input":"2022-10-08T20:20:43.674853Z","iopub.status.idle":"2022-10-08T20:20:43.869001Z","shell.execute_reply.started":"2022-10-08T20:20:43.674813Z","shell.execute_reply":"2022-10-08T20:20:43.867702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor value in ['banner','interstitial','rewarded']:\n    for fi in range(1,2):\n        col = f'shift{fi}WinBid_{value}'\n        d00[col] = d00g[value].fillna(method  = 'ffill')\n    d00.drop(columns = value,inplace = True)\nd00g = d00.groupby(did)\nfor value in ['banner','interstitial','rewarded']:\n    for fi in range(1,2):\n        col = f'shift{fi}WinBid_{value}'\n        d00[col] = d00g[col].shift(fi)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:20:43.870393Z","iopub.execute_input":"2022-10-08T20:20:43.870719Z","iopub.status.idle":"2022-10-08T20:21:14.462158Z","shell.execute_reply.started":"2022-10-08T20:20:43.870689Z","shell.execute_reply":"2022-10-08T20:21:14.461039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nd = d00.merge(d, on = [did,ets])\ndel d00\ndel d00g\nfor fi in range(1,2):\n    d[f'udtShift{fi}WinBid'] = 0\n    for value in ['banner','interstitial','rewarded']:\n        col = f'shift{fi}WinBid_{value}'\n        d.loc[d[udt].isin([value]),f'udtShift{fi}WinBid'] = d.loc[d[udt].isin([value]),col]\n        #d.drop(columns = col,inplace = True)\n        shift_udt_cls += [col]\n    gc.collect()\n    shift_udt_cls += [f'udtShift{fi}WinBid']\nprint(d.shape)    ","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:21:14.463776Z","iopub.execute_input":"2022-10-08T20:21:14.464293Z","iopub.status.idle":"2022-10-08T20:21:37.630215Z","shell.execute_reply.started":"2022-10-08T20:21:14.464205Z","shell.execute_reply":"2022-10-08T20:21:37.628916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# example\nd[shift_udt_cls+[did,udt,'winBid']].loc[d[did].isin([did0])]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:21:37.631733Z","iopub.execute_input":"2022-10-08T20:21:37.632051Z","iopub.status.idle":"2022-10-08T20:21:42.774208Z","shell.execute_reply.started":"2022-10-08T20:21:37.632023Z","shell.execute_reply":"2022-10-08T20:21:42.772911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## last winBid by c1","metadata":{}},{"cell_type":"code","source":"total_shift_udt_cls = []\ngrp = 'c1'\nd.sort_values(by = [did,grp,ets], inplace = True)\ndg = d.groupby([did,grp], as_index = False)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:21:42.805814Z","iopub.execute_input":"2022-10-08T20:21:42.806660Z","iopub.status.idle":"2022-10-08T20:21:55.869347Z","shell.execute_reply.started":"2022-10-08T20:21:42.806609Z","shell.execute_reply":"2022-10-08T20:21:55.868327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dg.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:21:55.870555Z","iopub.execute_input":"2022-10-08T20:21:55.870869Z","iopub.status.idle":"2022-10-08T20:21:58.430969Z","shell.execute_reply.started":"2022-10-08T20:21:55.870841Z","shell.execute_reply":"2022-10-08T20:21:58.429474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fi in range(1,6):    \n    col = f'shift_{grp}_{fi}WinBid'\n    d[col] = dg['winBid'].shift(fi)\n    total_shift_udt_cls.append(col)\n    #col += 'max'\n    #d[col] = dg['winBid'].shift(fi).cummax()\n    #total_shift_udt_cls.append(col)\n    #col += 'max'\n    #d[col] = dg['winBid'].shift(fi).cumcount()\n    #total_shift_udt_cls.append(col)\n    \n    #d[f'shift{fi}C4'] = dg['c4'].shift(fi)\n    #d[f'shift{fi}SentPrice'] = dg['sentPrice'].shift(fi)\n    #d[f'shift{fi}UnitDisplayType'] = dg['unitDisplayType'].shift(fi)\n    #d[f'shift{fi}Size'] = '' #dg['size'].shift(fi)\nprint(d.shape)\ndel dg\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:21:58.432553Z","iopub.execute_input":"2022-10-08T20:21:58.432905Z","iopub.status.idle":"2022-10-08T20:22:00.126664Z","shell.execute_reply.started":"2022-10-08T20:21:58.432872Z","shell.execute_reply":"2022-10-08T20:22:00.125489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#d = d.merge(d00, on = [grp,ets], how = 'left')\n#del d00, d00g\nd.sort_values(by = [did,ets],inplace = True)\nd.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:22:00.128546Z","iopub.execute_input":"2022-10-08T20:22:00.129359Z","iopub.status.idle":"2022-10-08T20:22:12.530745Z","shell.execute_reply.started":"2022-10-08T20:22:00.129315Z","shell.execute_reply":"2022-10-08T20:22:12.529671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d.columns","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:22:12.532687Z","iopub.execute_input":"2022-10-08T20:22:12.533144Z","iopub.status.idle":"2022-10-08T20:22:12.540674Z","shell.execute_reply.started":"2022-10-08T20:22:12.533099Z","shell.execute_reply":"2022-10-08T20:22:12.539388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Features: shifted ets and aggregated count,max,min","metadata":{}},{"cell_type":"code","source":"dg = d.groupby([did,P['GROUP']], as_index = False)\nd[f'shiftETS'] = dg[ets].shift(1)\nd['Count'] = dg['shift1WinBid'].cumcount()\nd['maxWinBid'] = dg['shift1WinBid'].cummax()\nd['minWinBid'] = dg['shift1WinBid'].cummin()\n#d['meanWinBid'] = dg['shift1WinBid'].expanding().mean()\nagg_cls = [fi + 'WinBid' for fi in ['max','min']] + ['Count','shiftETS']\ndel dg\ngc.collect()\n\ncnts_cls = []\nfor fi in range(2):\n    col = f'Count{fi}'\n    d[col] = 0 \n    d.loc[d['Count'] == fi,col] = 1\n    cnts_cls.append(col)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:22:12.542384Z","iopub.execute_input":"2022-10-08T20:22:12.542717Z","iopub.status.idle":"2022-10-08T20:22:17.848605Z","shell.execute_reply.started":"2022-10-08T20:22:12.542687Z","shell.execute_reply":"2022-10-08T20:22:17.847226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shift_cls = []\nfor col in []:#['brandName','correctModelName','connectionType','c1','osAndVersion','countryCode','bundleId','appVersion', 'mediationProviderVersion','unitDisplayType','c3']:\n    d.sort_values(by = [col,P['GROUP'],ets], inplace = True)\n    dg = d.groupby([col,P['GROUP']], as_index = False)\n    fi = 1\n    pcol = f'shift1WinBid{col}'\n    d[pcol] = dg['winBid'].shift(fi)\n    del dg \n    gc.collect()\n    shift_cls.append(pcol)\nd.sort_values(by = [did,ets], inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:22:17.852417Z","iopub.execute_input":"2022-10-08T20:22:17.852805Z","iopub.status.idle":"2022-10-08T20:22:30.164701Z","shell.execute_reply.started":"2022-10-08T20:22:17.852765Z","shell.execute_reply":"2022-10-08T20:22:30.163624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def area(s):\n    ss = s.split('x')\n    return int(ss[0]), int(ss[1]), int(ss[0])*int(ss[1])//10**3","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:22:30.165965Z","iopub.execute_input":"2022-10-08T20:22:30.166288Z","iopub.status.idle":"2022-10-08T20:22:30.171290Z","shell.execute_reply.started":"2022-10-08T20:22:30.166248Z","shell.execute_reply":"2022-10-08T20:22:30.170566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Other features based on public notebook, but I don't think it improves score:","metadata":{}},{"cell_type":"code","source":"%%time\n# https://www.kaggle.com/code/seraquevence/digturbine-lightgbmodel-v01\nd['dif_sent_floor'] = d['sentPrice'] - d['bidFloorPrice']\nd['div_sent_floor'] = d['sentPrice']/d['bidFloorPrice']\n#\nd['sum_c2_c4'] = d['c2'] + d['c4']\nd['dif_c2_c4'] = d['c2'] - d['c4']\nd['mult_c2_c4'] = d['c2'] * d['c4']\nd['div_c2_c4'] = d['c2'] / d['c4']\nd['1_c2'] = 1/d['c2']\nd['1_c4'] = 1/d['c4']\n#d['c1_c3'] = d['c1']+'_'+d['c3']","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:22:30.172294Z","iopub.execute_input":"2022-10-08T20:22:30.173209Z","iopub.status.idle":"2022-10-08T20:22:30.497434Z","shell.execute_reply.started":"2022-10-08T20:22:30.173179Z","shell.execute_reply":"2022-10-08T20:22:30.496114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Preparing for one-hot encoding, dict V for placehorder values:","metadata":{}},{"cell_type":"code","source":"V = {}\ncols = ['unitDisplayType','c3'] #+ [f'shift{fi}UnitDisplayType' for fi in range(1,6)]\nfor col in cols:\n    V[col] = [fi for fi in d[col].unique() if not pd.isnull(fi)]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:24:13.440739Z","iopub.execute_input":"2022-10-08T20:24:13.441203Z","iopub.status.idle":"2022-10-08T20:24:14.606114Z","shell.execute_reply.started":"2022-10-08T20:24:13.441168Z","shell.execute_reply":"2022-10-08T20:24:14.605068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Storing only grouped d, d will not be needed any more","metadata":{}},{"cell_type":"code","source":"%%time\ndg = d.groupby([did], as_index = False)\ndel d\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:24:14.607892Z","iopub.execute_input":"2022-10-08T20:24:14.608226Z","iopub.status.idle":"2022-10-08T20:24:14.747982Z","shell.execute_reply.started":"2022-10-08T20:24:14.608194Z","shell.execute_reply":"2022-10-08T20:24:14.746689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spliting data\nFunction presented below works fine for splitting data to train and test:","metadata":{}},{"cell_type":"code","source":"def split(dg):\n    test = dg.tail(1)\n    # ttrain = d.head(-1).copy() DOES NOT WORK!\n    train = dg.apply(lambda x: x.iloc[:-1])\n    \n    train.reset_index(drop = True, inplace = True)\n    test.reset_index(drop = True, inplace = True)\n    return train, test","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:24:14.749658Z","iopub.execute_input":"2022-10-08T20:24:14.750002Z","iopub.status.idle":"2022-10-08T20:24:14.758376Z","shell.execute_reply.started":"2022-10-08T20:24:14.749972Z","shell.execute_reply":"2022-10-08T20:24:14.756983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"all data splitted into train and test (its reversed operation of concatenation):","metadata":{}},{"cell_type":"code","source":"train, test = split(dg)\ndel dg\ngc.collect()\nprint(train.shape, test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:24:14.761321Z","iopub.execute_input":"2022-10-08T20:24:14.761902Z","iopub.status.idle":"2022-10-08T20:25:29.227035Z","shell.execute_reply.started":"2022-10-08T20:24:14.761866Z","shell.execute_reply":"2022-10-08T20:25:29.225536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So I made this step one more time:\n\ntrain data splitted into train_train and oof (this frame will be used only for evaluation):","metadata":{}},{"cell_type":"code","source":"train.sort_values(by = [did,ets], inplace = True)\ndg = train.groupby(did, as_index = False)\ndel train\ngc.collect()\nttrain, oof = split(dg)\ndel dg\ngc.collect()\nprint(ttrain.shape, oof.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:25:29.228902Z","iopub.execute_input":"2022-10-08T20:25:29.229290Z","iopub.status.idle":"2022-10-08T20:26:58.017234Z","shell.execute_reply.started":"2022-10-08T20:25:29.229240Z","shell.execute_reply":"2022-10-08T20:26:58.016156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Saving oof for further eval:","metadata":{}},{"cell_type":"code","source":"oof[['deviceId','winBid']].to_csv('oof_target.csv',index = False)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:26:58.018797Z","iopub.execute_input":"2022-10-08T20:26:58.019136Z","iopub.status.idle":"2022-10-08T20:26:58.144849Z","shell.execute_reply.started":"2022-10-08T20:26:58.019106Z","shell.execute_reply":"2022-10-08T20:26:58.143691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Going further with values for one-hot encoding:","metadata":{}},{"cell_type":"code","source":"for col in ['brandName','correctModelName','connectionType','c1','osAndVersion','size','countryCode','bundleId','appVersion', 'mediationProviderVersion']:\n    tc = test[col].value_counts()\n    V[col] = list(tc[tc/test.shape[0]>0.01].index)\n\nV['dayofweek'] =['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']\nO = {}\nfor col in V:\n    O[col] = OneHotEncoder(handle_unknown='ignore')\n    O[col].fit(pd.DataFrame(V[col]))","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:26:58.146807Z","iopub.execute_input":"2022-10-08T20:26:58.147536Z","iopub.status.idle":"2022-10-08T20:26:58.201210Z","shell.execute_reply.started":"2022-10-08T20:26:58.147492Z","shell.execute_reply":"2022-10-08T20:26:58.199890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ts_conv(ts):\n    t = datetime.utcfromtimestamp(int(ts/1000))\n    # year = t.strftime('%Y') all data is from 2022\n    month = t.strftime('%m') # all data is 06-07\n    day = t.strftime('%d')\n    hour = t.strftime('%H')\n    dayofweek = t.strftime('%A')\n    return int(month), int(day), int(hour), dayofweek","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:26:58.203049Z","iopub.execute_input":"2022-10-08T20:26:58.203435Z","iopub.status.idle":"2022-10-08T20:26:58.211662Z","shell.execute_reply.started":"2022-10-08T20:26:58.203401Z","shell.execute_reply":"2022-10-08T20:26:58.210125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"all_(features) converts dict of list to a list","metadata":{}},{"cell_type":"code","source":"def all_(features):\n    all_features = []\n    for fi in features:\n        all_features += features[fi]\n    return all_features","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:26:58.213128Z","iopub.execute_input":"2022-10-08T20:26:58.213865Z","iopub.status.idle":"2022-10-08T20:26:58.222419Z","shell.execute_reply.started":"2022-10-08T20:26:58.213828Z","shell.execute_reply":"2022-10-08T20:26:58.221463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_train = ttrain.groupby([did,P['GROUP']]).tail(5).copy() ","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:26:58.226293Z","iopub.execute_input":"2022-10-08T20:26:58.227220Z","iopub.status.idle":"2022-10-08T20:27:03.404045Z","shell.execute_reply.started":"2022-10-08T20:26:58.227180Z","shell.execute_reply":"2022-10-08T20:27:03.403029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_train.shape)\nprint(oof.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:03.405362Z","iopub.execute_input":"2022-10-08T20:27:03.405687Z","iopub.status.idle":"2022-10-08T20:27:03.412066Z","shell.execute_reply.started":"2022-10-08T20:27:03.405658Z","shell.execute_reply":"2022-10-08T20:27:03.410741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"preprocessing here means:\n* applying one-hot encoding\n* saving selected features to dict\n* saving for processed frames\n* splitting train data to folds (by deviceId) and saves to numpy arrays selected features","metadata":{}},{"cell_type":"code","source":"def preprocess(df, mode = 'train', fold = None):\n    features = {}\n\n    df['month'], df['day'], df['hour'], df['dayofweek'] = zip(*df.apply(lambda x: ts_conv(x.eventTimestamp), axis=1))\n    features['DATETIME'] = ['month','day','hour']\n    ##############\n    ## ONE-HOT ###\n    ##############\n    for col in V:\n        enc_df = pd.DataFrame(O[col].transform(df[[col]]).toarray().astype(np.uint8))\n        ec = list(O[col].categories_[0])\n        for fi in range(1,6):\n            if f'shift{fi}' in col:\n                print(ec)\n                cls = [f'shift{fi}_'+str(fe) for fe in ec ]\n                enc_df.columns = cls\n        #if 'shift' in col:\n        #    print(ec)\n        #    cls = ['shift_'+str(fi) for fi in ec ]\n        #    enc_df.columns = cls\n        if 'c4' == col:\n            cls = ['c4_'+str(fi) for fi in ec ]\n            enc_df.columns = cls\n        elif 'shift' not in col:\n            cls = ec\n            enc_df.columns = cls\n        df = df.join(enc_df)\n        features[col.upper()] = cls\n    \n    features['COUNTS'] = cnts_cls\n    features['OTHERS'] = [ets,'c2','bidFloorPrice','sentPrice','c4'] #[f'shift{fi}WinBid' for fi in range(1,6)]  +[f'shift{fi}C4' for fi in range(1,6)]\n    features['OTHERS'] += ['dif_sent_floor','div_sent_floor','sum_c2_c4','dif_c2_c4','mult_c2_c4','div_c2_c4'] # 'c1_c3'\n    features['OTHERS_AGG'] = agg_cls\n    features['C2_C4'] = ['1_c2','1_c4']\n    #features['SHIFTS'] = shift_cls\n    features['SHIFTS_UDT'] = shift_udt_cls\n    features['SHIFTS_WIN_BIDS'] = shift_win_bids\n    features['TOTAL_SHIFTS_UDT'] = total_shift_udt_cls\n    #features['SHIFTSENTPRICE'] = [f'shift{fi}SentPrice' for fi in range(1,6)]\n    df[ets] = (df[ets] - 1656290100421)//1000\n    df['shiftETS'] = (df['shiftETS'] - 1656290100421)//1000\n    \n    ###########\n    ## SAVE ###\n    ###########\n    \n    if mode == 'train':\n        kf = KFold(n_splits=P['NFOLDS'], random_state=0, shuffle=True)\n        print(df.shape)\n        print(all_(features))\n        for fi, dff in df.groupby(P['GROUP']):\n            \n            opath = f\"{P['GROUP']}/{fi}\"\n            os.makedirs(opath, exist_ok = True)\n            ddu = dff[did].unique()\n            for fold, (tr_id, val_id) in enumerate(kf.split(ddu)):\n                tr_d = [ddu[fi] for fi in tr_id]\n                val_d = [ddu[fi] for fi in val_id]\n                X_val = dff.loc[dff[did].isin(val_d)][all_(features)]\n                print(np.isnan(X_val).sum())\n                y_val = dff.loc[dff[did].isin(val_d)][targets]\n                print(X_val.shape)\n                with open(f'{opath}/fold{fold}.npy', 'wb') as f:\n                    np.save(f, X_val)\n                    np.save(f, y_val)\n                del X_val, y_val\n    else:\n        X = df.loc[:,all_(features)]\n        with open(f'{mode}.npy', 'wb') as f:\n            np.save(f, X)\n        del X\n    return features","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:03.414104Z","iopub.execute_input":"2022-10-08T20:27:03.414488Z","iopub.status.idle":"2022-10-08T20:27:03.435444Z","shell.execute_reply.started":"2022-10-08T20:27:03.414443Z","shell.execute_reply":"2022-10-08T20:27:03.433992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Applying preprocess","metadata":{}},{"cell_type":"code","source":"targets = ['winBid']\n\ntrain_train.reset_index(drop = True, inplace = True)\npreprocess(train_train,'train')\noof.reset_index(drop = True, inplace = True)\npreprocess(oof,'oof')\ntest.reset_index(drop = True, inplace = True)\nfeatures = preprocess(test,'test')\n#len(features)\nprint(features)\nP['FEATURES_DICT'] = {k: v for v, k in enumerate(all_(features))}","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:03.437413Z","iopub.execute_input":"2022-10-08T20:27:03.437773Z","iopub.status.idle":"2022-10-08T20:27:25.036765Z","shell.execute_reply.started":"2022-10-08T20:27:03.437741Z","shell.execute_reply":"2022-10-08T20:27:25.035393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Saving data-metadata to params:","metadata":{}},{"cell_type":"code","source":"P['FEATURES'] = features\nimport yaml\nwith open(r'params.yaml', 'w') as file:\n    documents = yaml.dump(P, file)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:25.038844Z","iopub.execute_input":"2022-10-08T20:27:25.039352Z","iopub.status.idle":"2022-10-08T20:27:25.141926Z","shell.execute_reply.started":"2022-10-08T20:27:25.039300Z","shell.execute_reply":"2022-10-08T20:27:25.140542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Saving dataframes to csv:","metadata":{}},{"cell_type":"code","source":"cdir = 'cdata'\nos.makedirs(cdir, exist_ok = True)\noof.to_csv(f'{cdir}/oof_data.csv',index = False)\ntest.to_csv(f'{cdir}/test_data.csv',index = False)\nimport shutil\n#shutil.copy(f'{idir}/test_data.csv',f'{cdir}/test_data.csv')\nshutil.copy(f'{idir}/sample_submission.csv',f'{cdir}/sample_submission.csv')\ntrain_train[[did]].to_csv(f'{cdir}/train_train_data.csv',index = False)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:25.143655Z","iopub.execute_input":"2022-10-08T20:27:25.144029Z","iopub.status.idle":"2022-10-08T20:27:27.971115Z","shell.execute_reply.started":"2022-10-08T20:27:25.143994Z","shell.execute_reply":"2022-10-08T20:27:27.969820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_train, oof, test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:27.973437Z","iopub.execute_input":"2022-10-08T20:27:27.973801Z","iopub.status.idle":"2022-10-08T20:27:28.135279Z","shell.execute_reply.started":"2022-10-08T20:27:27.973768Z","shell.execute_reply":"2022-10-08T20:27:28.133927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are ready - let's go!","metadata":{}},{"cell_type":"markdown","source":"# Model\nimporting packages:","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nfrom sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom xgboost import XGBRegressor\nimport lightgbm as lgb\n#from cuml.neighbors import KNeighborsRegressor","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-10-08T20:27:28.136481Z","iopub.execute_input":"2022-10-08T20:27:28.137172Z","iopub.status.idle":"2022-10-08T20:27:29.469635Z","shell.execute_reply.started":"2022-10-08T20:27:28.137138Z","shell.execute_reply":"2022-10-08T20:27:29.467969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Params dict for training:\n* MODE = I tried autoML from h2o, xgb, lightgbm, elastic net, k-nn. At this moment LGBM works the best\n* STEPS = number of steps for backward feature selection\n* MAX_DEPTH, MAX_RUNTIME_SECS - for autoML\n* N_ESTIMATORS for xgb\n* INPUT path to transform output\n* EXCLUDED_LIST - excluded variables via feature selection\n* FEATURES - selected groups of features\n* OTHERS - overriding params['FEAUTRES']['OTHERS']","metadata":{}},{"cell_type":"code","source":"P = {}\nP['NFOLDS'] = 5\nP['FOLDS'] = [2,3,4]\nP['FOLDS'] = range(P['NFOLDS'])\nP['MODE'] = 'LGBM' # ENET XGB LGB AML True\nP['BOOSTING_TYPE'] = 'gbdt'#'dart'\n\nP['MAX_DEPTH'] = 6\nP['MAX_RUNTIME_SECS'] = 90 # max \nP['N_ESTIMATORS'] = 40\nP['TARGET'] = 'winBid'\nP['INPUT'] = './'\nP['STEPS'] = 1\nP['N_TRIALS'] = 1000# optuna\nP['N_TRIALS2'] = 0#\nP['EXCLUDED_LIST'] = ['c2',\n 'day',\n 'minWinBid',\n '11.3.3',\n 'Android-8.1.0',\n 'shift4WinBid',\n '397',\n '4e',\n '2.0',\n 'Huawei',\n 'shift1_banner',\n 'PH',\n 'dovi.coalmining.inc',\n '728x90',\n 'shift1_320x50',\n 'MX',\n 'com.YayySAL.DodgeAgent',\n 'banner',\n 'com.AppIdeas.LevelUpRunner',\n '3dc',\n 'iPad',\n '320x480',\n '0.32',\n '59b','ad3','c17','6b','f8d','fdc',\n 'PL']\n\nP['FEATURES'] = [\n #'APPVERSION',\n 'BRANDNAME',\n 'BUNDLEID',\n 'C1',\n 'C3',\n #'C2_C4',\n #'C4',\n 'CONNECTIONTYPE',\n #'CORRECTMODELNAME',\n 'COUNTRYCODE',\n 'DATETIME',\n 'MEDIATIONPROVIDERVERSION',\n #'OSANDVERSION',\n #'C2_CAT',\n #'C4_CAT',\n 'OTHERS',\n 'OTHERS_AGG',\n #'COUNTS',\n #'SIZE',\n #'SHIFTSENTPRICE',\n #'SHIFTS',\n #'SHIFTS_UDT',\n 'SHIFTS_WIN_BIDS',\n 'TOTAL_SHIFTS_UDT',\n 'UNITDISPLAYTYPE' ] #+ #[ f'SHIFT{fi}SIZE' for fi in range(1,2)] + [ f'SHIFT{fi}UNITDISPLAYTYPE' for fi in range(1,6)]#,\n #'DAYOFWEEK'] #'DAYOFWEEK'\nets = 'eventTimestamp'\nP['OTHERS'] = ['c2', 'bidFloorPrice', 'sentPrice']#,'c2','c4','area','size0']#, 'shift1WinBid', 'shift2WinBid', 'shift3WinBid', 'shift4WinBid', 'shift5WinBid']\n#P['OTHERS'] = params['FEATURES']['SHIFTS_WIN_BIDS'][1:]\n#P['OTHERS'] += ['c4']\n#P['OTHERS'] += ['shift1C4',  'shift2C4', 'shift3C4', 'shift4C4', 'shift5C4']\n#P['OTHERS'] += ['dif_sent_floor','div_sent_floor','sum_c2_c4','dif_c2_c4','mult_c2_c4','div_c2_c4','c1_c3']\n#P['GROUP'] = 'UNITDISPLAYTYPE'\n#P['SHIFTS_UDT'] = ['udtShift1WinBid']\nP['TOTAL_SHIFTS_UDT'] = ['shift_c1_1WinBid','shift_c1_2WinBid']\n#P['COUNTS'] = ['Count1']\n#['']\n#P['SHIFTS_WIN_BIDS'] = ['shift1WinBid', 'shift2WinBid', 'shift3WinBid', 'shift4WinBid', 'shift5WinBid', 'shift6WinBid', 'shift7WinBid', 'shift8WinBid', 'shift9WinBid']\n#P['C2_C4'] = ['1_c2','1_c4']","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.471403Z","iopub.execute_input":"2022-10-08T20:27:29.471791Z","iopub.status.idle":"2022-10-08T20:27:29.483574Z","shell.execute_reply.started":"2022-10-08T20:27:29.471757Z","shell.execute_reply":"2022-10-08T20:27:29.482621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading params dict from Transform output","metadata":{}},{"cell_type":"code","source":"import yaml\nwith open(P['INPUT']+'params.yaml') as f:\n    params = yaml.full_load(f)\nprint(params['DATAVERSION'])\nprint(params['DATACOMMENT'])\nfor fi in params['FEATURES']:\n    print(fi +',')","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.484948Z","iopub.execute_input":"2022-10-08T20:27:29.485547Z","iopub.status.idle":"2022-10-08T20:27:29.541691Z","shell.execute_reply.started":"2022-10-08T20:27:29.485512Z","shell.execute_reply":"2022-10-08T20:27:29.540338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_features = []\nfor fi in P['FEATURES']:\n    all_features += params['FEATURES'][fi]\nprint(len(all_features))\nfeatures_dict = params['FEATURES_DICT']#{k: v for v, k in enumerate(all_features)}","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.543002Z","iopub.execute_input":"2022-10-08T20:27:29.544208Z","iopub.status.idle":"2022-10-08T20:27:29.552148Z","shell.execute_reply.started":"2022-10-08T20:27:29.544161Z","shell.execute_reply":"2022-10-08T20:27:29.551056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Selector feature ids for numpy arrays:","metadata":{}},{"cell_type":"code","source":"#all_features_sorted = [fi for fi in {k: v for k, v in sorted(features_dict.items(), key=lambda item: item[1])}]\nfeatures_ids = []\nfeatures = []\nfor fi in P['FEATURES']:\n    try:\n        features_ids += [features_dict[ffi] for ffi in P[fi]]\n        features += P[fi]\n        #if fi == params['GROUP']:\n        #    features_group = [features_dict[ffi] for ffi in P[fi]]\n    except:\n        print(fi)\n        features_ids += [features_dict[ffi] for ffi in params['FEATURES'][fi]]\n        features += params['FEATURES'][fi]\n        #if fi == params['GROUP']:\n        #    features_group = [features_dict[ffi] for ffi in params['FEATURES'][fi]]\nprint(len(features_ids))\nprint(len(features))","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.553487Z","iopub.execute_input":"2022-10-08T20:27:29.554530Z","iopub.status.idle":"2022-10-08T20:27:29.565282Z","shell.execute_reply.started":"2022-10-08T20:27:29.554495Z","shell.execute_reply":"2022-10-08T20:27:29.564049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if P['MODE'] == 'AML':\n    import h2o\n    print(h2o.__version__)\n    from h2o.automl import H2OAutoML\n\n    h2o.init(max_mem_size='16G')","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.566518Z","iopub.execute_input":"2022-10-08T20:27:29.567086Z","iopub.status.idle":"2022-10-08T20:27:29.578636Z","shell.execute_reply.started":"2022-10-08T20:27:29.566915Z","shell.execute_reply":"2022-10-08T20:27:29.577533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Metric helper function for pandas:","metadata":{}},{"cell_type":"code","source":"def rmse(d,col, verbose):\n    r = ((d.winBid - d[col]) ** 2).mean() ** .5\n    if verbose: print(r)\n    return r","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.580337Z","iopub.execute_input":"2022-10-08T20:27:29.580867Z","iopub.status.idle":"2022-10-08T20:27:29.589851Z","shell.execute_reply.started":"2022-10-08T20:27:29.580836Z","shell.execute_reply":"2022-10-08T20:27:29.588562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#features_ids[27] # C28 features_dict{'col' : features_ids[27]}","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.591065Z","iopub.execute_input":"2022-10-08T20:27:29.591901Z","iopub.status.idle":"2022-10-08T20:27:29.601326Z","shell.execute_reply.started":"2022-10-08T20:27:29.591866Z","shell.execute_reply":"2022-10-08T20:27:29.600478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading proper train/val split and features for each group:","metadata":{}},{"cell_type":"code","source":"def load_fold(fold,group,value):\n    with open(P['INPUT']+ f'/{group}/{value}/fold{fold}.npy', 'rb') as f:\n        X_train = np.load(f)[:,features_ids]\n        y_train = np.load(f)#[:,1:]\n    #return np.hstack((X_train,y_train,np.ones((X_train.shape[0],1))*fold))\n    return X_train, y_train\n\ndef load_train_val(fold,group,value):\n    X_val, y_val = load_fold(fold,group,value)\n    X_train = np.vstack([load_fold(fi,group,value)[0] for fi in range(P['NFOLDS']) if fi not in [fold]])\n    y_train = np.vstack([load_fold(fi,group,value)[1] for fi in range(P['NFOLDS']) if fi not in [fold]])\n    return X_train, y_train, X_val, y_val","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.602446Z","iopub.execute_input":"2022-10-08T20:27:29.603570Z","iopub.status.idle":"2022-10-08T20:27:29.621736Z","shell.execute_reply.started":"2022-10-08T20:27:29.603535Z","shell.execute_reply":"2022-10-08T20:27:29.620336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T20:27:29.623206Z","iopub.execute_input":"2022-10-08T20:27:29.623583Z","iopub.status.idle":"2022-10-08T20:27:29.775224Z","shell.execute_reply.started":"2022-10-08T20:27:29.623552Z","shell.execute_reply":"2022-10-08T20:27:29.774134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main training loop:\n* if STEPS == 1 no feature selection applied\n* iterates over values in P['GROUP'] ('size' in this version)\n* iterates over each fold\n* current parameters for lgbm has been adjusted manually","metadata":{}},{"cell_type":"code","source":"cur_score = 100\nexcluded_list = P['EXCLUDED_LIST']#['c2','day']#['c2']\nafeatures = [fi for fi in features if fi not in excluded_list]\nfor step in range(P['STEPS']):\n    random.shuffle(afeatures)\n    for excluded_variable in afeatures:\n        if step == 0: excluded_variable = ''\n        #print(excluded_variable)\n        #def submission(mode = 'test'):\n\n        features_ids = []\n        features = []\n        for fi in P['FEATURES']:\n            try:\n                features_ids += [features_dict[ffi] for ffi in P[fi] if ffi != excluded_variable and ffi not in excluded_list]\n                features += [ffi for ffi in P[fi]  if ffi != excluded_variable and ffi not in excluded_list]\n            except:\n                features_ids += [features_dict[ffi] for ffi in params['FEATURES'][fi] if ffi != excluded_variable and ffi not in excluded_list]\n                features += [ffi for ffi in params['FEATURES'][fi]  if ffi != excluded_variable and ffi not in excluded_list]\n        #print(len(features_ids))\n        #print(len(features))\n        with open(P['INPUT']+ f'oof.npy', 'rb') as f:\n            X = np.load(f)[:,features_ids]\n\n        with open(P['INPUT']+ f'test.npy', 'rb') as f:\n            X_test = np.load(f)[:,features_ids]\n\n        if P['MODE'] == 'AML': \n            ho = h2o.H2OFrame(X)\n            ht = h2o.H2OFrame(X_test)\n\n        oof = pd.read_csv(P['INPUT'] + '/oof_target.csv')\n        test = pd.read_csv(P['INPUT'] + '/cdata/test_data.csv')\n\n        VERBOSE = -1\n        group = params['GROUP']\n\n        if group != '':\n            for value in os.listdir(P['INPUT']+'/'+group):\n                if VERBOSE>0:\n                    print('############')\n                    print(value)\n                    print('############')\n                for fold in P['FOLDS']:\n                    if VERBOSE>0: print(f'FOLD: {fold}')\n                        #if fold > 0: continue\n                    pcol = f'preds_{group}_{value}_fold{fold}'\n                    X_train, y_train, X_val, y_val = load_train_val(fold,group,value)\n\n                    if P['MODE'] == 'AML':\n                        p = .7\n                        Xy = np.hstack((X_train, y_train**p))\n                        del X_train, y_train, X_val, y_val\n                        hf = h2o.H2OFrame(Xy)\n\n                        for ff in range(hf.shape[1]):\n                            hd = hf[ff].dtype\n                            if hd == np.dtype('<U'):\n                                print(ff)\n                                hf[ff] = hf[ff].ascharacter().asnumeric()\n                        hf.columns[-1]\n                        aml = H2OAutoML(max_models=10000, seed=47, max_runtime_secs=P['MAX_RUNTIME_SECS'])\n                        aml.train(x=hf.columns[:-1], y=hf.columns[-1], training_frame=hf)\n                        del Xy\n                        del hf\n                        lb = aml.leaderboard\n                        lb.head(rows=lb.nrows)  \n\n                        ### OOF ###\n                        preds = aml.predict(ho)\n                        #pcol = f'preds_fold{fold}'\n                        #oof[pcol] = np.exp(preds['predict'].as_data_frame().values.squeeze())\n                        #oof[pcol] = preds['predict'].as_data_frame().values.squeeze()\n                        oof[pcol] = preds['predict'].as_data_frame().values.squeeze()**(1/p)\n\n                        ### TEST ####\n\n                        preds = aml.predict(ht)\n                        test[pcol] = preds['predict'].as_data_frame().values.squeeze()**(1/p)\n\n                        del aml\n                    elif P['MODE'] == 'XGB':\n                        n_estimators = 400\n                        a = 1\n                        learning_rate = 6/n_estimators\n                        model = XGBRegressor(base_score = 0.1, verbosity = 0, tree_method = 'gpu_hist', gpu_id = 0,\n                                         max_depth = P['MAX_DEPTH'], n_estimators = n_estimators, early_stopping_rounds = 2, \n                                             learning_rate = learning_rate, gamma = 2,feature_selector = 'greedy')\n                        model.fit(X_train, y_train*a, eval_set = [(X_val, y_val*a)],verbose = False)\n                        #model.save_model(f'model_fold{fold}.json')\n                        ### OOF ###\n                        preds = model.predict(X)\n                        #pcol = f'preds_fold{fold}'\n                        preds[preds<0.01] = 0.01 \n                        oof[pcol] = preds.squeeze()/a\n                        ### TEST ###\n                        preds = model.predict(X_test)\n\n                        preds[preds<0.01] = 0.01 \n                        test[pcol] = preds.squeeze()/a\n                    elif P['MODE'] == 'LGBM':\n                        hyper_params = {\n                    'task': 'train',\n                    'boosting_type': P['BOOSTING_TYPE'],\n                    'objective': 'regression',\n                    'metric': 'rmse',\n                    'learning_rate': 0.15,\n                    'feature_fraction': .9,\n                    'bagging_fraction': .7,\n                    'bagging_freq': 10,\n                    'verbose': -1,\n                    \"max_depth\": 11,\n                    \"num_iterations\": 2000\n                }\n                        a = 1\n                        model = lgb.LGBMRegressor(**hyper_params)\n                        #logger = \n                        p = .7\n                        model.fit(X_train, y_train.squeeze()**p,\n                            eval_set=[(X_val, y_val.squeeze()**p)],\n                            eval_metric='rmse',\n                            #early_stopping_rounds=25,\n                                  callbacks = [lgb.log_evaluation(1000),lgb.early_stopping(300,verbose = False)])\n                        \n                        ### OOF ###\n                        preds = model.predict(X)\n                        #pcol = f'preds_fold{fold}'\n                        preds[preds<0.01] = 0.01 \n                        oof[pcol] = preds.squeeze()**(1/p)/a\n                        ### TEST ###\n                        preds = model.predict(X_test)\n                        #pcol = f'preds_fold{fold}'\n                        preds[preds<0.01] = 0.01 \n                        test[pcol] = preds.squeeze()**(1/p)/a\n                    elif P['MODE'] == 'ENET':\n                        a = 1\n                        model = ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3)\n                        model.fit(np.nan_to_num(X_train,-1), y_train)\n                        preds = model.predict(np.nan_to_num(X,-1))\n                        #pcol = f'preds_fold{fold}'\n                        preds[preds<0.01] = 0.01 \n                        oof[pcol] = preds.squeeze()/a\n                        ### TEST ###\n                        preds = model.predict(np.nan_to_num(X_test,-1))\n                        #pcol = f'preds_fold{fold}'\n                        preds[preds<0.01] = 0.01 \n                        test[pcol] = preds.squeeze()/a\n                    else:# P['MODE'] = 'KNN':\n                        model = KNeighborsRegressor(n_neighbors=150, p = 2, algo = 'brute',leaf_size = 60)\n                        model.fit(X_train, y_train)\n\n                        a = 1\n                        ### OOF ###\n                        preds = model.predict(X)\n                        #pcol = f'preds_fold{fold}'\n                        preds[preds<0.01] = 0.01 \n                        oof[pcol] = preds.squeeze()/a\n                        ### TEST ###\n                        preds = model.predict(X_test)\n                        #pcol = f'preds_fold{fold}'\n                        preds[preds<0.01] = 0.01 \n                        test[pcol] = preds.squeeze()/a\n                    #del X_train, y_train\n                    #rmse(oof,pcol)\n        did = 'deviceId'\n        oofd = pd.read_csv(P['INPUT'] + '/cdata/oof_data.csv')\n        oofb = oofd.merge(oof,on = [did,'winBid'])\n\n        for value, df in oofb.groupby(group):\n            cname = f'{value}_fold'\n            #print(value)\n            #print(df.shape)\n            #eval(df,value)\n            oofb.loc[oofb[group].isin([value]),'WinBid'] = oofb.loc[oofb[group].isin([value])].filter(like = cname, axis = 1).mean(axis =1)\n            test.loc[test[group].isin([value]),'WinBid'] = test.loc[test[group].isin([value])].filter(like = cname, axis = 1).mean(axis =1)\n        \n        score = rmse(oofb,'WinBid',0)\n        if score <= cur_score:\n            cur_score = score\n            if step == 0:\n                print(f'first step! {score}')\n            else:\n                print(f'step {step} improvement! {score}; removed {excluded_variable}')\n                \n                afeatures.remove(excluded_variable)\n                excluded_list += [excluded_variable]\n            break\n\n        else:\n            print(f'score {score}; keep {excluded_variable}, ')","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:19:50.895432Z","iopub.execute_input":"2022-10-08T21:19:50.895970Z","iopub.status.idle":"2022-10-08T21:20:42.949661Z","shell.execute_reply.started":"2022-10-08T21:19:50.895927Z","shell.execute_reply":"2022-10-08T21:20:42.948657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Postprocessing\n* I've found this operation works fine for target and sweet-spot value for p around .7-.8\n\n>                         p = .7\n                        model.fit(X_train, y_train.squeeze()**p,\n                            eval_set=[(X_val, y_val.squeeze()**p)],\n\n* values below 0.01 should be updated\n\n> preds[preds<0.01] = 0.01 \n\n* if winBid for size 1024 is lower than 480 then update:\n\n>         for fi in ['768x1024','320x480']: \n            oofb[f'{fi}_WinBid'] = oofb.filter(like = fi, axis = 1).mean(axis =1) \n            test[f'{fi}_WinBid'] = test.filter(like = fi, axis = 1).mean(axis =1) \n        cond = oofb['size'].isin(['768x1024']) \n        oofb.loc[cond,'WinBid'] = oofb.loc[cond,['768x1024_WinBid','320x480_WinBid']].max(axis = 1) \n        \n* averaging preds using mean, in previous versions median also works fine\n","metadata":{}},{"cell_type":"markdown","source":"## Post1 = optuna for folds","metadata":{}},{"cell_type":"code","source":"import optuna\nfrom optuna.samplers import TPESampler\nif P['N_TRIALS'] >0:\n    optuna.logging.set_verbosity(optuna.logging.WARNING)\n    group = params['GROUP']\n    R1 = {}\n    if group != '':\n        for value in os.listdir(P['INPUT']+'/'+group):\n            print(group, value)\n            oo = oofb.loc[oofb[group].isin([value])].copy()\n            pcol = f'preds_{group}_{value}_fold'\n            print()\n            def objective(trial):\n                oo['obj'] = 0 \n                for feature in oo.filter(like = pcol, axis = 1).columns:\n                    oo['obj'] += trial.suggest_float(feature, 0, 1)*oo[feature]\n                return rmse(oo,'obj',0)\n            sampler = TPESampler(seed=2)\n            study = optuna.create_study(sampler=sampler)\n            study.optimize(objective, n_trials=P['N_TRIALS'])\n            R1[value] = study.best_params\n            print(study.best_value)\n    def post1(d):\n        d['WinBid'] = 0 \n        for value in os.listdir(P['INPUT']+'/'+group):\n            pcol = f'preds_{group}_{value}_fold'\n            for feature in oo.filter(like = pcol, axis = 1).columns:\n                cond = d[group].isin([value])\n                d.loc[cond,'WinBid'] += R1[value][feature]*d.loc[cond,feature]\n        return d\n    #opt(test,R)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:20:42.951616Z","iopub.execute_input":"2022-10-08T21:20:42.951982Z","iopub.status.idle":"2022-10-08T21:23:41.342200Z","shell.execute_reply.started":"2022-10-08T21:20:42.951942Z","shell.execute_reply":"2022-10-08T21:23:41.341149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if P['N_TRIALS'] >0:\n    rmse(post1(oofb),'WinBid',1)\n    post1(test)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.343670Z","iopub.execute_input":"2022-10-08T21:23:41.344586Z","iopub.status.idle":"2022-10-08T21:23:41.536069Z","shell.execute_reply.started":"2022-10-08T21:23:41.344543Z","shell.execute_reply":"2022-10-08T21:23:41.534875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking if technically everything is ok:","metadata":{}},{"cell_type":"code","source":"oofb['winBid'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.539605Z","iopub.execute_input":"2022-10-08T21:23:41.540107Z","iopub.status.idle":"2022-10-08T21:23:41.549036Z","shell.execute_reply.started":"2022-10-08T21:23:41.540053Z","shell.execute_reply":"2022-10-08T21:23:41.547778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Features importance:","metadata":{}},{"cell_type":"code","source":"if P['MODE'] == 'LGBM':\n    fi = pd.DataFrame({\"imp\": model.feature_importances_, \"feature\": features})\n    fi.sort_values(by = 'imp',ascending = False,inplace = True)\nfi","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.550850Z","iopub.execute_input":"2022-10-08T21:23:41.551349Z","iopub.status.idle":"2022-10-08T21:23:41.572969Z","shell.execute_reply.started":"2022-10-08T21:23:41.551280Z","shell.execute_reply":"2022-10-08T21:23:41.571685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi['feature'][50:].values.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.574854Z","iopub.execute_input":"2022-10-08T21:23:41.575633Z","iopub.status.idle":"2022-10-08T21:23:41.586135Z","shell.execute_reply.started":"2022-10-08T21:23:41.575570Z","shell.execute_reply":"2022-10-08T21:23:41.585003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', 40)\noofb.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.588003Z","iopub.execute_input":"2022-10-08T21:23:41.588472Z","iopub.status.idle":"2022-10-08T21:23:41.639978Z","shell.execute_reply.started":"2022-10-08T21:23:41.588418Z","shell.execute_reply":"2022-10-08T21:23:41.638667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Post 2","metadata":{}},{"cell_type":"code","source":"import optuna\nif P['N_TRIALS2'] >0:\n    optuna.logging.set_verbosity(optuna.logging.WARNING)\n    group = 'Count'\n    R2 = {}\n    if group != '':\n        for value in [0,1]:\n            print(group, value)\n            oo = oofb.loc[oofb[group].isin([value])].copy()\n            #pcol = f'preds_{group}_{value}_fold'\n            print()\n            def objective(trial):\n                oo['obj'] = 0 \n                cond = (oo['WinBid']>trial.suggest_float('a', 1, 50)) & (oo['Count'] == value )\n                #print(cond.sum())\n                oo.loc[cond,'obj'] = oo.loc[cond,'WinBid']*trial.suggest_float('b',1,2)\n                return rmse(oo,'obj',0)\n\n            study = optuna.create_study()\n            study.optimize(objective, n_trials=P['N_TRIALS2'])\n            R2[value] = study.best_params\n            print(study.best_value)\n    def post2(d):\n        #d['WinBid'] = 0 \n        for value in [0,1]:\n            cond = (d['WinBid']>R2[value]['a'])  & (d['Count'] == value )\n            d.loc[cond,'WinBid'] = d.loc[cond,'WinBid']*R2[value]['b']\n        return d","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.641425Z","iopub.execute_input":"2022-10-08T21:23:41.642641Z","iopub.status.idle":"2022-10-08T21:23:41.654091Z","shell.execute_reply.started":"2022-10-08T21:23:41.642605Z","shell.execute_reply":"2022-10-08T21:23:41.652865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if P['N_TRIALS2'] >0:\n    rmse(post2(oofb),'WinBid',1)\n    post2(test)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.655654Z","iopub.execute_input":"2022-10-08T21:23:41.656692Z","iopub.status.idle":"2022-10-08T21:23:41.674576Z","shell.execute_reply.started":"2022-10-08T21:23:41.656654Z","shell.execute_reply":"2022-10-08T21:23:41.673390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Post3","metadata":{}},{"cell_type":"code","source":"def post3(d):\n    for fi in ['768x1024','320x480']: \n        d[f'{fi}_WinBid'] = d.filter(like = fi, axis = 1).mean(axis =1) \n    cond2 = (d['size'].isin(['768x1024'])) & (d['768x1024_WinBid'] > 100)\n    d['320x480_WinBid2'] = d['320x480_WinBid']*1.1 # :) :)\n    d.loc[cond2,'WinBid'] = d.loc[cond2,['768x1024_WinBid','320x480_WinBid','320x480_WinBid2']].max(axis = 1) \n\npost3(oofb)\npost3(test)\nrmse(oofb,'WinBid',1)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.678368Z","iopub.execute_input":"2022-10-08T21:23:41.679296Z","iopub.status.idle":"2022-10-08T21:23:41.770015Z","shell.execute_reply.started":"2022-10-08T21:23:41.679241Z","shell.execute_reply":"2022-10-08T21:23:41.768952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Post 4","metadata":{}},{"cell_type":"code","source":"\ndef post4(d):\n    cond2 = d['WinBid'] == d['WinBid']\n    d.loc[cond2,'WinBid'] = d.loc[cond2,['WinBid','sentPrice']].max(axis = 1) \npost4(oofb)\npost4(test)\nrmse(oofb,'WinBid',1)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.771362Z","iopub.execute_input":"2022-10-08T21:23:41.771672Z","iopub.status.idle":"2022-10-08T21:23:41.832265Z","shell.execute_reply.started":"2022-10-08T21:23:41.771645Z","shell.execute_reply":"2022-10-08T21:23:41.831051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Error grouped by uDT, count, size\nAuctions for new devices usually comes with rewarded, interstitial, 480 or 1024 - that is one of reason why these auctions has huge squared error.\nAnother is simply winBid is much higher that udt=banner (placed usually on small size, for existing devices).","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_columns', 10)\npd.set_option('display.max_rows', 200)\noofb['se'] = (oofb['winBid'] - oofb['WinBid']) **2","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.833643Z","iopub.execute_input":"2022-10-08T21:23:41.833968Z","iopub.status.idle":"2022-10-08T21:23:41.841198Z","shell.execute_reply.started":"2022-10-08T21:23:41.833939Z","shell.execute_reply":"2022-10-08T21:23:41.839980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"udt = 'unitDisplayType'\noofb.groupby(udt).mean()['se']","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.842828Z","iopub.execute_input":"2022-10-08T21:23:41.843211Z","iopub.status.idle":"2022-10-08T21:23:41.900353Z","shell.execute_reply.started":"2022-10-08T21:23:41.843177Z","shell.execute_reply":"2022-10-08T21:23:41.899493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oofb.groupby('Count').mean()['se'].head()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.901790Z","iopub.execute_input":"2022-10-08T21:23:41.902145Z","iopub.status.idle":"2022-10-08T21:23:41.945245Z","shell.execute_reply.started":"2022-10-08T21:23:41.902114Z","shell.execute_reply":"2022-10-08T21:23:41.944030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oofb.groupby('size').mean()['se']","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.946467Z","iopub.execute_input":"2022-10-08T21:23:41.946790Z","iopub.status.idle":"2022-10-08T21:23:41.990110Z","shell.execute_reply.started":"2022-10-08T21:23:41.946761Z","shell.execute_reply":"2022-10-08T21:23:41.989224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oofb.groupby(['c1','size']).mean()['se']","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:41.991567Z","iopub.execute_input":"2022-10-08T21:23:41.991891Z","iopub.status.idle":"2022-10-08T21:23:42.041869Z","shell.execute_reply.started":"2022-10-08T21:23:41.991861Z","shell.execute_reply":"2022-10-08T21:23:42.040726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oofb.groupby('c3').mean()['se']","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:42.043651Z","iopub.execute_input":"2022-10-08T21:23:42.044021Z","iopub.status.idle":"2022-10-08T21:23:42.086696Z","shell.execute_reply.started":"2022-10-08T21:23:42.043989Z","shell.execute_reply":"2022-10-08T21:23:42.085468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oofb['winBid'] = oofb['WinBid']\ntest['winBid'] = test['WinBid']\ntest[[did,'winBid']].head()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:42.088313Z","iopub.execute_input":"2022-10-08T21:23:42.088766Z","iopub.status.idle":"2022-10-08T21:23:42.103577Z","shell.execute_reply.started":"2022-10-08T21:23:42.088724Z","shell.execute_reply":"2022-10-08T21:23:42.102447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Saving submission:","metadata":{}},{"cell_type":"code","source":"#test['winBid'] = test.filter(like = 'fold',axis = 1).median(axis = 1)\noofb[[did,'winBid']].to_csv('oof.csv',index = False)\ntest[[did,'winBid']].to_csv('submission.csv',index = False)\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-08T21:23:42.105357Z","iopub.execute_input":"2022-10-08T21:23:42.105750Z","iopub.status.idle":"2022-10-08T21:23:42.316530Z","shell.execute_reply.started":"2022-10-08T21:23:42.105718Z","shell.execute_reply":"2022-10-08T21:23:42.315507Z"},"trusted":true},"execution_count":null,"outputs":[]}]}