{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\nDEBUG = False","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-15T13:17:03.465504Z","iopub.execute_input":"2022-08-15T13:17:03.466227Z","iopub.status.idle":"2022-08-15T13:17:03.492049Z","shell.execute_reply.started":"2022-08-15T13:17:03.466119Z","shell.execute_reply":"2022-08-15T13:17:03.491211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/train_data.csv')\ntest  = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/test_data.csv')\n\nif DEBUG:\n    train = train.sample(0.1)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T13:17:03.506661Z","iopub.execute_input":"2022-08-15T13:17:03.507192Z","iopub.status.idle":"2022-08-15T13:17:33.751955Z","shell.execute_reply.started":"2022-08-15T13:17:03.507155Z","shell.execute_reply":"2022-08-15T13:17:33.750812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T13:17:33.753728Z","iopub.execute_input":"2022-08-15T13:17:33.754067Z","iopub.status.idle":"2022-08-15T13:17:33.791165Z","shell.execute_reply.started":"2022-08-15T13:17:33.754037Z","shell.execute_reply":"2022-08-15T13:17:33.790291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-15T13:17:33.792356Z","iopub.execute_input":"2022-08-15T13:17:33.792836Z","iopub.status.idle":"2022-08-15T13:17:33.799772Z","shell.execute_reply.started":"2022-08-15T13:17:33.792805Z","shell.execute_reply":"2022-08-15T13:17:33.798986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Repartition of Features","metadata":{}},{"cell_type":"code","source":"for c in train.columns:\n    df = train[c]\n    print(f'number of unique values for feature {c}: {len(pd.DataFrame(df).value_counts().index)}')\n    display(pd.DataFrame(df).value_counts().iloc[:5])","metadata":{"execution":{"iopub.status.busy":"2022-08-15T13:21:59.957311Z","iopub.execute_input":"2022-08-15T13:21:59.957707Z","iopub.status.idle":"2022-08-15T13:22:44.888587Z","shell.execute_reply.started":"2022-08-15T13:21:59.957674Z","shell.execute_reply":"2022-08-15T13:22:44.887647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# study most numerous id","metadata":{}},{"cell_type":"code","source":"top_10_id = train.deviceId.value_counts()[:10]","metadata":{"execution":{"iopub.status.busy":"2022-08-15T13:17:33.802185Z","iopub.execute_input":"2022-08-15T13:17:33.802961Z","iopub.status.idle":"2022-08-15T13:17:34.840345Z","shell.execute_reply.started":"2022-08-15T13:17:33.802919Z","shell.execute_reply":"2022-08-15T13:17:34.839408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = train[train.deviceId == top_10_id.index[0]]","metadata":{"execution":{"iopub.status.busy":"2022-08-15T13:17:34.841939Z","iopub.execute_input":"2022-08-15T13:17:34.842790Z","iopub.status.idle":"2022-08-15T13:17:35.400541Z","shell.execute_reply.started":"2022-08-15T13:17:34.842751Z","shell.execute_reply":"2022-08-15T13:17:35.399571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = device.sort_values(by='eventTimestamp').reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T13:17:35.401735Z","iopub.execute_input":"2022-08-15T13:17:35.402052Z","iopub.status.idle":"2022-08-15T13:17:35.426024Z","shell.execute_reply.started":"2022-08-15T13:17:35.402023Z","shell.execute_reply":"2022-08-15T13:17:35.425126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Price driver:","metadata":{}},{"cell_type":"code","source":"for c in ['bundleId', 'appVersion', 'countryCode', 'osAndVersion', 'connectionType', 'c1', 'c2', 'c3', 'c4', 'size']:\n    print(c)\n    print(np.unique(device[c]))\n    device.groupby(c)['winBid'].plot()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:43:16.318808Z","iopub.execute_input":"2022-08-14T10:43:16.319217Z","iopub.status.idle":"2022-08-14T10:43:19.117717Z","shell.execute_reply.started":"2022-08-14T10:43:16.319182Z","shell.execute_reply":"2022-08-14T10:43:19.116170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# we see that c1 and size partition the data ... maybe c4 too\n\nNot sure what we can do with c1 as it is anonymised.\nSize should be in square inch.","metadata":{}},{"cell_type":"code","source":"np.unique(train['c1'])","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:43:19.119291Z","iopub.execute_input":"2022-08-14T10:43:19.119798Z","iopub.status.idle":"2022-08-14T10:43:26.609251Z","shell.execute_reply.started":"2022-08-14T10:43:19.119765Z","shell.execute_reply":"2022-08-14T10:43:26.607585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(test['c1'])","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:43:26.610773Z","iopub.execute_input":"2022-08-14T10:43:26.611127Z","iopub.status.idle":"2022-08-14T10:43:26.648366Z","shell.execute_reply.started":"2022-08-14T10:43:26.611097Z","shell.execute_reply":"2022-08-14T10:43:26.646553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"at least we have no suprise with new c1 in test","metadata":{}},{"cell_type":"code","source":"np.unique(train['size'])","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:43:26.652585Z","iopub.execute_input":"2022-08-14T10:43:26.653967Z","iopub.status.idle":"2022-08-14T10:43:34.209325Z","shell.execute_reply.started":"2022-08-14T10:43:26.653920Z","shell.execute_reply":"2022-08-14T10:43:34.207984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I was thinking about changing size to number of pixels but we only have a small number. llgbm should handle this without problem.","metadata":{}},{"cell_type":"code","source":"def calc_area(df):\n    size = df['size'].str.split('x',expand=True)\n    df['area'] = size[0].astype(int) *  size[1].astype(int)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:43:34.210981Z","iopub.execute_input":"2022-08-14T10:43:34.211739Z","iopub.status.idle":"2022-08-14T10:43:34.218571Z","shell.execute_reply.started":"2022-08-14T10:43:34.211696Z","shell.execute_reply":"2022-08-14T10:43:34.217193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = calc_area(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:43:34.220059Z","iopub.execute_input":"2022-08-14T10:43:34.220370Z","iopub.status.idle":"2022-08-14T10:43:55.647029Z","shell.execute_reply.started":"2022-08-14T10:43:34.220342Z","shell.execute_reply":"2022-08-14T10:43:55.645372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Device ID study","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib_venn import venn2","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:43:55.648799Z","iopub.execute_input":"2022-08-14T10:43:55.649295Z","iopub.status.idle":"2022-08-14T10:43:55.848460Z","shell.execute_reply.started":"2022-08-14T10:43:55.649260Z","shell.execute_reply":"2022-08-14T10:43:55.847082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"venn2([set(np.unique(train.deviceId)),set(np.unique(test.deviceId))], set_labels = ('train', 'test'))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:43:55.850256Z","iopub.execute_input":"2022-08-14T10:43:55.852186Z","iopub.status.idle":"2022-08-14T10:44:07.782702Z","shell.execute_reply.started":"2022-08-14T10:43:55.852117Z","shell.execute_reply":"2022-08-14T10:44:07.780596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 1410 Id in test that are not in train.","metadata":{}},{"cell_type":"code","source":"diff_set = set(np.unique(test.deviceId)) - set(np.unique(train.deviceId))\n\nlist(diff_set)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:44:07.785606Z","iopub.execute_input":"2022-08-14T10:44:07.786757Z","iopub.status.idle":"2022-08-14T10:44:19.992172Z","shell.execute_reply.started":"2022-08-14T10:44:07.786677Z","shell.execute_reply":"2022-08-14T10:44:19.990379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# target encoding","metadata":{}},{"cell_type":"code","source":"price_by_group = train.groupby(['c1','size']).winBid.mean()\nprice_by_group1 = train.groupby(['deviceId','c1','size']).winBid.mean()\nprice_by_group14 = train.groupby(['deviceId','c1','c4','size']).winBid.mean()\nprice_by_group_no_id14 = train.groupby(['c1','c4','size']).winBid.mean()\nprice_by_group_no_id1 = train.groupby(['c1','size']).winBid.mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:44:19.994473Z","iopub.execute_input":"2022-08-14T10:44:19.995983Z","iopub.status.idle":"2022-08-14T10:44:30.836526Z","shell.execute_reply.started":"2022-08-14T10:44:19.995925Z","shell.execute_reply":"2022-08-14T10:44:30.834895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"price_by_group.name = 'winBid0'\nprice_by_group1.name = 'winBid1'\nprice_by_group14.name = 'winBid14'\nprice_by_group_no_id14.name = 'winBid_no_id14'\nprice_by_group_no_id1.name = 'winBid_no_id1'","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:44:30.838863Z","iopub.execute_input":"2022-08-14T10:44:30.839396Z","iopub.status.idle":"2022-08-14T10:44:30.847797Z","shell.execute_reply.started":"2022-08-14T10:44:30.839346Z","shell.execute_reply":"2022-08-14T10:44:30.845948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_price =  test.merge(price_by_group, left_on = ['c1','size'], right_index=True, how='left')\ntest_price =  test_price.merge(price_by_group1, left_on = ['deviceId','c1','size'], right_index=True, how='left')\ntest_price =  test_price.merge(price_by_group14, left_on = ['deviceId','c1','c4','size'], right_index=True, how='left')\ntest_price =  test_price.merge(price_by_group_no_id14, left_on = ['c1','c4','size'], right_index=True, how='left')\ntest_price =  test_price.merge(price_by_group_no_id1, left_on = ['c1','size'], right_index=True, how='left')","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:44:30.849756Z","iopub.execute_input":"2022-08-14T10:44:30.850763Z","iopub.status.idle":"2022-08-14T10:44:31.236428Z","shell.execute_reply.started":"2022-08-14T10:44:30.850685Z","shell.execute_reply":"2022-08-14T10:44:31.234908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"code","source":"test_price['is_in_train'] = test.deviceId.isin(np.unique(train.deviceId))\ntest_price['sub'] = test_price.winBid0 #np.where(test_price.winBid1.isna(),test_price.winBid_no_id14,test_price.winBid14)\ntest_price['sub'] = test_price[['sub','sentPrice']].max(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:44:31.238516Z","iopub.execute_input":"2022-08-14T10:44:31.239187Z","iopub.status.idle":"2022-08-14T10:44:31.248619Z","shell.execute_reply.started":"2022-08-14T10:44:31.239131Z","shell.execute_reply":"2022-08-14T10:44:31.246906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = test_price[['deviceId','sub']]\nsub.columns = ['deviceId','winBid']\n\nsub.to_csv(\"bidFloor_submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-14T10:44:31.250363Z","iopub.status.idle":"2022-08-14T10:44:31.251557Z","shell.execute_reply.started":"2022-08-14T10:44:31.251206Z","shell.execute_reply":"2022-08-14T10:44:31.251255Z"},"trusted":true},"execution_count":null,"outputs":[]}]}