{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np \nimport pandas as pd\nfrom datetime import datetime\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport time\n\nimport gc\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor\nfrom math import sqrt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-20T12:11:14.123237Z","iopub.execute_input":"2023-06-20T12:11:14.123762Z","iopub.status.idle":"2023-06-20T12:11:16.951425Z","shell.execute_reply.started":"2023-06-20T12:11:14.123718Z","shell.execute_reply":"2023-06-20T12:11:16.950393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ydata-profiling","metadata":{"execution":{"iopub.status.busy":"2023-06-20T09:38:51.120754Z","iopub.execute_input":"2023-06-20T09:38:51.121739Z","iopub.status.idle":"2023-06-20T09:39:16.666366Z","shell.execute_reply.started":"2023-06-20T09:38:51.121695Z","shell.execute_reply":"2023-06-20T09:39:16.664963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Produce and save the profiling report\nfrom ydata_profiling import ProfileReport","metadata":{"execution":{"iopub.status.busy":"2023-06-20T09:39:16.668816Z","iopub.execute_input":"2023-06-20T09:39:16.669630Z","iopub.status.idle":"2023-06-20T09:39:19.343060Z","shell.execute_reply.started":"2023-06-20T09:39:16.669579Z","shell.execute_reply":"2023-06-20T09:39:19.342009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-20T12:11:20.977424Z","iopub.execute_input":"2023-06-20T12:11:20.977810Z","iopub.status.idle":"2023-06-20T12:11:20.986891Z","shell.execute_reply.started":"2023-06-20T12:11:20.977781Z","shell.execute_reply":"2023-06-20T12:11:20.985670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/train_data.csv')\ntest = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/test_data.csv')\nsample = pd.read_csv('../input/digital-turbine-auction-bid-price-prediction/sample_submission.csv')\n\nprint(f'Number of rows in train set: {train.shape[0]}')\nprint(f'Number of rows in test set: {test.shape[0]}')\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:11:21.740340Z","iopub.execute_input":"2023-06-20T12:11:21.741241Z","iopub.status.idle":"2023-06-20T12:12:03.647576Z","shell.execute_reply.started":"2023-06-20T12:11:21.741197Z","shell.execute_reply":"2023-06-20T12:12:03.646635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deviceId_val_counts = train.deviceId.value_counts()\ndeviceId_val_counts","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:12:03.649121Z","iopub.execute_input":"2023-06-20T12:12:03.649700Z","iopub.status.idle":"2023-06-20T12:12:05.513291Z","shell.execute_reply.started":"2023-06-20T12:12:03.649670Z","shell.execute_reply":"2023-06-20T12:12:05.511919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ts_conv(ts):\n    t = datetime.utcfromtimestamp(int(ts/1000))\n    # year = t.strftime('%Y') all data is from 2022\n    month = t.strftime('%m') # all data is 06-07\n    day = t.strftime('%d')\n    hour = t.strftime('%H')\n    return month, day, hour\n\n\ndef preprocess_timestamp(d):\n    d['month'], d['day'], d['hour'] = zip(*d.apply(lambda x: ts_conv(x[\"eventTimestamp\"]), axis=1))\n    return d","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:12:05.514663Z","iopub.execute_input":"2023-06-20T12:12:05.515063Z","iopub.status.idle":"2023-06-20T12:12:05.523629Z","shell.execute_reply.started":"2023-06-20T12:12:05.515033Z","shell.execute_reply":"2023-06-20T12:12:05.521598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = preprocess_timestamp(train)\ntest = preprocess_timestamp(test)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:12:05.527214Z","iopub.execute_input":"2023-06-20T12:12:05.527611Z","iopub.status.idle":"2023-06-20T12:15:28.495574Z","shell.execute_reply.started":"2023-06-20T12:12:05.527579Z","shell.execute_reply":"2023-06-20T12:15:28.494300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# there are only june and july included\ntrain[\"month\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:15:28.509899Z","iopub.execute_input":"2023-06-20T12:15:28.510282Z","iopub.status.idle":"2023-06-20T12:15:29.898190Z","shell.execute_reply.started":"2023-06-20T12:15:28.510254Z","shell.execute_reply":"2023-06-20T12:15:29.896806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# there are only june and july included\ntrain[\"day\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:15:29.911130Z","iopub.execute_input":"2023-06-20T12:15:29.911886Z","iopub.status.idle":"2023-06-20T12:15:31.286752Z","shell.execute_reply.started":"2023-06-20T12:15:29.911851Z","shell.execute_reply":"2023-06-20T12:15:31.285836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:15:31.287755Z","iopub.execute_input":"2023-06-20T12:15:31.288121Z","iopub.status.idle":"2023-06-20T12:15:31.324941Z","shell.execute_reply.started":"2023-06-20T12:15:31.288091Z","shell.execute_reply":"2023-06-20T12:15:31.324040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Comparison of the test and train data - do the distributions of the variables look similar?","metadata":{}},{"cell_type":"markdown","source":"#### To speed up the analysis of datasets, there is used data profiling. It can speed up the process of catching the data imbalances, inconsistencies etc. (ydata-profiling has automatic detection of potential data quality issues.)","metadata":{}},{"cell_type":"code","source":"profile_test = ProfileReport(test, title=\"Test data Profile Report\")\nprofile_test.to_file(\"report_test.html\")","metadata":{"execution":{"iopub.status.busy":"2023-06-20T09:43:31.593193Z","iopub.execute_input":"2023-06-20T09:43:31.593638Z","iopub.status.idle":"2023-06-20T09:44:20.675296Z","shell.execute_reply.started":"2023-06-20T09:43:31.593599Z","shell.execute_reply":"2023-06-20T09:44:20.674065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nThe ydata profiling give us an overview on the data and also autodetect problems within. The overall alerts are:\n1. high cardinality of brandName, appVersion, correctModelName, countryCode, deviceId, osAndVersion\n2.high correlation of:\n- c1 and: UnitDisplayType, bundleId, c3 and bidFloorprice\n- c3 and c1, bundleId, osVersionType\n- unitDisplayType is highly overall correlated with c1 and size\n- bundleId is highly overall correlated with c1 and c3, mediationProviderVersion, bidFloorPrice\n- osAndVersion is highly overall correlated with c3\tHigh correlation\n- c1 and: UnitDisplayType, bundleId, c3 and bidFloorprice\n- c3 and c1, bundleId, osVersionType\n- size is highly overall correlated with unitDisplayType\n- mediationProviderVersion is highly overall correlated with bundleId\n- bidFloorPrice is highly overall correlated with bundleId and c1\n3. There are high imbalances in some of the features.","metadata":{}},{"cell_type":"code","source":"from IPython.display import HTML\nHTML(filename='/kaggle/working/report_test.html')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T09:44:20.677102Z","iopub.execute_input":"2023-06-20T09:44:20.678674Z","iopub.status.idle":"2023-06-20T09:44:20.753862Z","shell.execute_reply.started":"2023-06-20T09:44:20.678622Z","shell.execute_reply":"2023-06-20T09:44:20.752860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unfortunately the calculations of corelations in the train test was not working (supposedly it was due to the size of the file)\nprofile_train = ProfileReport(train, title=\"Train data Profile Report\", \n  correlations=None)\nprofile_train.to_file(\"report_train.html\")","metadata":{"execution":{"iopub.status.busy":"2023-06-20T09:44:20.754992Z","iopub.execute_input":"2023-06-20T09:44:20.755406Z","iopub.status.idle":"2023-06-20T10:00:37.043469Z","shell.execute_reply.started":"2023-06-20T09:44:20.755372Z","shell.execute_reply":"2023-06-20T10:00:37.042131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HTML(filename='/kaggle/working/report_train.html')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:00:37.045717Z","iopub.execute_input":"2023-06-20T10:00:37.046628Z","iopub.status.idle":"2023-06-20T10:00:37.146408Z","shell.execute_reply.started":"2023-06-20T10:00:37.046583Z","shell.execute_reply":"2023-06-20T10:00:37.144752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualization of comparison between the train and test data samples","metadata":{}},{"cell_type":"code","source":"def vc(col):\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))\n    train_sc = train[col].value_counts().sort_index(ascending=False).rename(f'Train {col} per device')\n    ax1.pie(train_sc, labels=train_sc.index)\n    ax1.set_title(train_sc.name)\n    test_sc = test[col].value_counts().sort_index(ascending=False).rename(f'Test {col} per device')\n    ax2.pie(test_sc, labels=test_sc.index)\n    ax2.set_title(test_sc.name)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:00:37.148256Z","iopub.execute_input":"2023-06-20T10:00:37.148637Z","iopub.status.idle":"2023-06-20T10:00:37.157796Z","shell.execute_reply.started":"2023-06-20T10:00:37.148606Z","shell.execute_reply":"2023-06-20T10:00:37.156884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fi in test.columns:\n    if test[fi].dtype in [np.int64, np.float64]:\n        print(f'numeric variable {fi}')\n    elif fi == 'deviceId':\n        print(f'special object variable {fi}')\n    else:\n        print(f'object variable {fi}')\n        vc(fi)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:00:37.159243Z","iopub.execute_input":"2023-06-20T10:00:37.159630Z","iopub.status.idle":"2023-06-20T10:01:55.873180Z","shell.execute_reply.started":"2023-06-20T10:00:37.159598Z","shell.execute_reply":"2023-06-20T10:01:55.871797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observations:\n\n1. unitDisplayType, bundleId, appVersion, countryCode, c1, size, mediationProviderVersion  has significantly different distr between train and test split\n2. The probable reason behind that, are the auctions per deviceid compared to test data. \n","metadata":{}},{"cell_type":"code","source":"test.groupby('deviceId')['brandName'].count().reset_index().sort_values(by = 'brandName', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:01:55.874684Z","iopub.execute_input":"2023-06-20T10:01:55.875082Z","iopub.status.idle":"2023-06-20T10:01:55.980631Z","shell.execute_reply.started":"2023-06-20T10:01:55.875049Z","shell.execute_reply":"2023-06-20T10:01:55.979416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('deviceId')['brandName'].count().reset_index().sort_values(by='brandName', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:01:55.984514Z","iopub.execute_input":"2023-06-20T10:01:55.984915Z","iopub.status.idle":"2023-06-20T10:01:58.245366Z","shell.execute_reply.started":"2023-06-20T10:01:55.984882Z","shell.execute_reply":"2023-06-20T10:01:58.244162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:01:58.247181Z","iopub.execute_input":"2023-06-20T10:01:58.247879Z","iopub.status.idle":"2023-06-20T10:01:58.255879Z","shell.execute_reply.started":"2023-06-20T10:01:58.247835Z","shell.execute_reply":"2023-06-20T10:01:58.254583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train.merge(test, on='deviceId'))","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:01:58.257754Z","iopub.execute_input":"2023-06-20T10:01:58.258574Z","iopub.status.idle":"2023-06-20T10:02:25.784332Z","shell.execute_reply.started":"2023-06-20T10:01:58.258529Z","shell.execute_reply":"2023-06-20T10:02:25.783187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.has_won.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:02:25.785665Z","iopub.execute_input":"2023-06-20T10:02:25.786000Z","iopub.status.idle":"2023-06-20T10:02:25.862510Z","shell.execute_reply.started":"2023-06-20T10:02:25.785973Z","shell.execute_reply":"2023-06-20T10:02:25.861172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizations of events per time","metadata":{}},{"cell_type":"code","source":"fig = plt.subplots(nrows=1, figsize=(15,10))\nsns.histplot(train.eventTimestamp)\nsns.histplot(test.eventTimestamp)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T10:02:25.863955Z","iopub.execute_input":"2023-06-20T10:02:25.864597Z","iopub.status.idle":"2023-06-20T10:02:35.971142Z","shell.execute_reply.started":"2023-06-20T10:02:25.864566Z","shell.execute_reply":"2023-06-20T10:02:35.969968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All of the devices in training deviceId's are included in test sample. The test set has all of the last events for each device (the highest number in timestamp). ","metadata":{}},{"cell_type":"markdown","source":"### Data Transformation","metadata":{}},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:15:31.326524Z","iopub.execute_input":"2023-06-20T12:15:31.326843Z","iopub.status.idle":"2023-06-20T12:15:31.497809Z","shell.execute_reply.started":"2023-06-20T12:15:31.326818Z","shell.execute_reply":"2023-06-20T12:15:31.496520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['test'] = 0\ntest['test'] = 1\ndf = pd.concat([train,test])\ndel test","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:15:31.499303Z","iopub.execute_input":"2023-06-20T12:15:31.499663Z","iopub.status.idle":"2023-06-20T12:15:34.929340Z","shell.execute_reply.started":"2023-06-20T12:15:31.499631Z","shell.execute_reply":"2023-06-20T12:15:34.927936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt = df.winBid.value_counts().reset_index()\ncnt.columns = [\"bid\", \"cnt\"]\ncnt","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:15:34.930876Z","iopub.execute_input":"2023-06-20T12:15:34.931236Z","iopub.status.idle":"2023-06-20T12:15:35.187811Z","shell.execute_reply.started":"2023-06-20T12:15:34.931205Z","shell.execute_reply":"2023-06-20T12:15:35.186617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['eventDate'] = pd.to_datetime(df['eventTimestamp'], unit='ms')\ndf['hour'] = df['eventDate'].dt.hour\ndf['day_of_week'] = df['eventDate'].dt.dayofweek - 27\ndf['is_weekend'] = np.where(df['eventDate'].dt.dayofweek > 4, 1, 0)\n    \ndf.sort_values(by=[\"deviceId\", \"eventTimestamp\"], inplace=True,ignore_index=True)\ndf['timestamp_lagged'] = df.loc[:, ['deviceId', 'eventTimestamp']].groupby('deviceId').rank(ascending=False)['eventTimestamp'].astype(np.int32) - 1\ndf[\"winBid_lagged\"] = df.loc[:, ['deviceId', 'winBid']].groupby('deviceId').shift(1)['winBid']\ndf[\"count_trans\"] = df.loc[:, ['deviceId', 'eventTimestamp']].groupby('deviceId').rank()['eventTimestamp'].astype(np.int32)\n# min max, etc can be added","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:43:18.130019Z","iopub.execute_input":"2023-06-20T12:43:18.130573Z","iopub.status.idle":"2023-06-20T12:43:45.218750Z","shell.execute_reply.started":"2023-06-20T12:43:18.130530Z","shell.execute_reply":"2023-06-20T12:43:45.217546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sentPrice Features\ndf['sentPrice_lagged'] = df.loc[:, ['deviceId', 'sentPrice']].groupby('deviceId').shift(1)['sentPrice']\ndf['sentPrice_mean'] = df.loc[:, ['deviceId', 'sentPrice_lagged']].groupby('deviceId').cumsum()['sentPrice_lagged'] # Firstly calc all of the sent pricese and divide by number of transactions\ndf['sentPrice_mean'] = df['sentPrice_mean'] / df['count_trans']\ndf['sentPrice_min'] = df.loc[:, ['deviceId', 'sentPrice_lagged']].groupby('deviceId').cummin()['sentPrice_lagged']\ndf['sentPrice_max'] = df.loc[:, ['deviceId', 'sentPrice_lagged']].groupby('deviceId').cummax()['sentPrice_lagged']\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:43:45.220915Z","iopub.execute_input":"2023-06-20T12:43:45.221302Z","iopub.status.idle":"2023-06-20T12:43:54.982488Z","shell.execute_reply.started":"2023-06-20T12:43:45.221270Z","shell.execute_reply":"2023-06-20T12:43:54.981322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# winBid features\ndf['winBid_mean'] = df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').cumsum()['winBid_lagged']\ndf['winBid_mean'] = df['winBid_mean'] / df['count_trans']\ndf['winBid_min'] = df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').cummin()['winBid_lagged']\ndf['winBid_max'] = df.loc[:, ['deviceId', 'winBid_lagged']].groupby('deviceId').cummax()['winBid_lagged']\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:43:54.984580Z","iopub.execute_input":"2023-06-20T12:43:54.985104Z","iopub.status.idle":"2023-06-20T12:44:01.678449Z","shell.execute_reply.started":"2023-06-20T12:43:54.985059Z","shell.execute_reply":"2023-06-20T12:44:01.677091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"winBid_mean_c1 = df.groupby('c1')[\"winBid\"].mean()\nwinBid_mean_c3 = df.groupby('c3')[\"winBid\"].mean()\n\ndf['winBid_mean_c1'] = df['c1'].map(winBid_mean_c1)\ndf['winBid_mean_c3'] = df['c3'].map(winBid_mean_c3)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:01.681347Z","iopub.execute_input":"2023-06-20T12:44:01.681721Z","iopub.status.idle":"2023-06-20T12:44:05.472936Z","shell.execute_reply.started":"2023-06-20T12:44:01.681690Z","shell.execute_reply":"2023-06-20T12:44:05.471871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Categorical variables","metadata":{}},{"cell_type":"code","source":"df['date'] = pd.to_datetime(df['eventTimestamp'], unit='ms')\ndf['is_US'] = np.where(df['countryCode'] == 'US', 1, 0)\ndf['is_JP'] = np.where(df['countryCode'] == 'JP', 1, 0)\ndf['is_CH'] = np.where(df['countryCode'] == 'CH', 1, 0)\ndf['is_BR'] = np.where(df['countryCode'] == 'BR', 1, 0)\ndf['is_weekend'] = np.where(df['date'].dt.dayofweek>4, 1, 0)\n\ndf['is_banner'] = np.where(df['unitDisplayType'] == 'banner', 1, 0)\ndf['is_interstitial'] = np.where(df['unitDisplayType'] == 'interstitial', 1, 0)\ndf['is_rewarded'] = np.where(df['unitDisplayType'] == 'rewarded', 1, 0)\ndf['is_WIFI'] = np.where(df['connectionType'] == 'WIFI', 1, 0) # There is 5% unknown\ndf['is_3G'] = np.where(df['connectionType'] == '3G', 1, 0)\n\ndf['is_apple'] = np.where(df['brandName'] == 'Apple', 1, 0)\ndf['is_android'] = np.where(df['osAndVersion'].str.split('-').str[0] == \"Android\", 1, 0)\ndf['c2_c4'] = df['c2'] / df['c4']","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:05.474149Z","iopub.execute_input":"2023-06-20T12:44:05.475019Z","iopub.status.idle":"2023-06-20T12:44:48.647581Z","shell.execute_reply.started":"2023-06-20T12:44:05.474985Z","shell.execute_reply":"2023-06-20T12:44:48.646383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Modelling\nFor the fist step, I picked the LightGBM. This is the gradient boosting method like XGBoost, CatBoost. Although CatBoost is supporting also the text columns and also in comparison to the other one it is symmetric tree algorithm (faster computing, control overfitting).\nIn the next step CatBoost will be tested.","metadata":{}},{"cell_type":"code","source":"params = {\n    'task': 'train',\n    'boosting_type': 'gbdt', # try later dart\n    'objective': 'rmse',\n    'learning_rate': 0.1,\n    'bagging_fraction': 0.65,\n    'metric': 'rmse',\n    'num_iterations': 500,\n    'verbose': -1,\n    'bagging_fraction': 0.8\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:48.649382Z","iopub.execute_input":"2023-06-20T12:44:48.649902Z","iopub.status.idle":"2023-06-20T12:44:48.656027Z","shell.execute_reply.started":"2023-06-20T12:44:48.649857Z","shell.execute_reply":"2023-06-20T12:44:48.655062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Features to train\nFEATURES = ['is_US', \"is_JP\", \"is_CH\", 'is_BR', 'is_weekend', 'is_banner', 'is_interstitial', 'is_rewarded', 'is_WIFI', 'is_3G', 'is_apple', 'is_android',\n'hour', 'day_of_week',\n'winBid_mean', 'winBid_max', 'winBid_min',\n'sentPrice_lagged', 'sentPrice_mean', 'sentPrice_max', 'sentPrice_min','count_trans', 'c2_c4', \"winBid_mean_c1\", \"winBid_mean_c3\"]\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:48.657525Z","iopub.execute_input":"2023-06-20T12:44:48.657893Z","iopub.status.idle":"2023-06-20T12:44:48.669953Z","shell.execute_reply.started":"2023-06-20T12:44:48.657862Z","shell.execute_reply":"2023-06-20T12:44:48.668786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.c3","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:48.671359Z","iopub.execute_input":"2023-06-20T12:44:48.671712Z","iopub.status.idle":"2023-06-20T12:44:48.687795Z","shell.execute_reply.started":"2023-06-20T12:44:48.671675Z","shell.execute_reply":"2023-06-20T12:44:48.686458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.c2","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:48.689081Z","iopub.execute_input":"2023-06-20T12:44:48.689476Z","iopub.status.idle":"2023-06-20T12:44:48.703300Z","shell.execute_reply.started":"2023-06-20T12:44:48.689446Z","shell.execute_reply":"2023-06-20T12:44:48.701867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.c4","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:48.708016Z","iopub.execute_input":"2023-06-20T12:44:48.708502Z","iopub.status.idle":"2023-06-20T12:44:48.720289Z","shell.execute_reply.started":"2023-06-20T12:44:48.708468Z","shell.execute_reply":"2023-06-20T12:44:48.719179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# K Fold split\nmodels = []\nimportances = []\npredictions = []\noofs = []\n\nfolds = KFold(n_splits = 9, shuffle = True, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:48.721395Z","iopub.execute_input":"2023-06-20T12:44:48.721768Z","iopub.status.idle":"2023-06-20T12:44:48.734723Z","shell.execute_reply.started":"2023-06-20T12:44:48.721739Z","shell.execute_reply":"2023-06-20T12:44:48.733196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = df[df[\"test\"] == 0]\ntest_df = df[df[\"test\"] == 1]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:48.736679Z","iopub.execute_input":"2023-06-20T12:44:48.737024Z","iopub.status.idle":"2023-06-20T12:44:55.356569Z","shell.execute_reply.started":"2023-06-20T12:44:48.736998Z","shell.execute_reply":"2023-06-20T12:44:55.355045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:20:45.619665Z","iopub.execute_input":"2023-06-20T12:20:45.621586Z","iopub.status.idle":"2023-06-20T12:20:59.086945Z","shell.execute_reply.started":"2023-06-20T12:20:45.621535Z","shell.execute_reply":"2023-06-20T12:20:59.085728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold, (train_idx, valid_idx) in enumerate(folds.split(train_df, train_df['winBid'])):\n    start = time.time()\n    print(f'--------- Fold: {fold} ----------')\n    \n    train_df['winBid']\n    X_train, y_train = train_df[FEATURES].iloc[train_idx], train_df['winBid'].iloc[train_idx]\n    X_valid, y_valid = train_df[FEATURES].iloc[valid_idx],  train_df['winBid'].iloc[valid_idx]\n\n    model = lgb.LGBMRegressor(**params)\n\n    model.fit(X_train[FEATURES], y_train, eval_set = (X_valid[FEATURES], y_valid),\n          callbacks = [lgb.early_stopping(40), lgb.log_evaluation(100)])\n    \n    oof = model.predict(X_valid[FEATURES], num_iteration = model.best_iteration_)\n    fold_score = sqrt(mean_squared_error(oof, y_valid))\n    print(f'RMSE OFF of this fold is: {fold_score:0.2f}')\n    \n    importance = pd.DataFrame(index = model.feature_name_, data = model.feature_importances_, columns = [f'{fold}_importance'])\n    fold_test_pred = model.predict(test_df[FEATURES], num_iteration = model.best_iteration_)\n\n    predictions.append(fold_test_pred)\n    models.append(model)\n    oofs.append(fold_score)\n    importances.append(importance)\n    \n    end = time.time()\n    full_time = (end - start)\n    \n    print(f'Time to train, predict and save results: {full_time:0.2f} seconds')","metadata":{"execution":{"iopub.status.busy":"2023-06-20T12:44:55.358215Z","iopub.execute_input":"2023-06-20T12:44:55.359394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Summary of results\nprint(f'General CV (OOF): {np.mean(oofs):0.2f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.float_format', lambda x: '%.3f' % x)\nimportances_df = pd.concat(importances, axis = 1)\nimportances_df['Feature'] = importance.index\nimportances_df['Value'] = importances_df['0_importance'] + importances_df['1_importance'] + importances_df['2_importance'] + importances_df['3_importance'] + \\\n                          importances_df['4_importance'] + importances_df['5_importance'] + importances_df['6_importance'] \nimportances_df = importances_df.sort_values('Value', ascending = False)\nimportances_df['Value'] = importances_df['Value'] / importances_df['Value'].sum()\n\nplt.rcParams['figure.dpi'] = 400\nfig = plt.figure(figsize = (5, 3), facecolor = 'white')\ngs = fig.add_gridspec(1, 2)\ngs.update(wspace = 0.3, hspace = 0)\nax0 = fig.add_subplot(gs[0, 0])\nax0.set_facecolor(\"white\")\nfor s in [\"right\", \"top\"]:\n    ax0.spines[s].set_visible(False)\n\nax0.text(0, -1.8, 'Feature importance ', color = 'black', fontsize = 7.3, ha = 'left', va = 'bottom', weight = 'bold')\nax0.text(0, -1.8, 'final lightGBM model', color = 'black', fontsize = 6, ha = 'left', va = 'top')\nax0_sns = sns.barplot(ax = ax0, y = importances_df['Feature'], x = importances_df['Value'], edgecolor = \"black\", linewidth = 0.3, color = \"#225478\", orient = \"h\")\nax0_sns.set_ylabel(\"Feature name\", fontsize = 4.5)\nax0_sns.set_xlabel(\"Feature importance (7 folds, ratio of total)\", fontsize = 4.5)\nax0_sns.tick_params(labelsize = 3.5)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Summary\n\nIn the next step, I would try to create more features, check whether to eliminate features that are highly correlated with each other. Then try the CatBoost and XGBoost models. \n\n1. gbdt also works much better in those results (lower OOF error  in comparison to the dart)\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}