{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_df = pd.read_csv('../input/facebook-recruiting-iv-human-or-bot/bids.csv.zip')\ntrain_df = pd.read_csv('../input/facebook-recruiting-iv-human-or-bot/train.csv.zip')\ntest_df = pd.read_csv('../input/facebook-recruiting-iv-human-or-bot/test.csv.zip')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Cleaning","metadata":{}},{"cell_type":"code","source":"bids_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Missing values","metadata":{}},{"cell_type":"code","source":"bids_df.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_df.isnull().sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_percent = bids_df['country'].isnull().mean()\nprint(f\"Percentage of missing data in country column: {missing_percent*100: .2f}%\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We find that only the country column has missing values and it is only a small proportion of the entire dataset.","metadata":{}},{"cell_type":"code","source":"unique_countries = bids_df['country'].value_counts()\n\nfig, ax = plt.subplots(figsize = (18, 4.8))\ncountries_distribution = sns.barplot(x = unique_countries.index, y = unique_countries.values, ax = ax)                        \nax.get_xaxis().set_visible(False)\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We check the distribution of the countries and find that there is one country that is significantly higher than the rest. Hence, mode imputation may be a good approach.","metadata":{}},{"cell_type":"code","source":"bids_df['country'] = bids_df['country'].fillna(bids_df['country'].mode()[0])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"### Features of unique counts using .nunique()","metadata":{}},{"cell_type":"code","source":"bidder_unique = bids_df.groupby(\"bidder_id\").nunique().reset_index()\nbidder_unique.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_df.merge(bidder_unique, on='bidder_id', how='left').drop(columns = ['payment_account', 'address'], errors = 'ignore')\ntrain_set = train_set.rename(columns = {'bid_id': 'num_bids', 'auction': 'num_auct', 'merchandise': 'num_merch_type', 'device': 'num_device_type', 'time': 'num_time', 'country': 'num_ctry', 'ip': 'num_ip', 'url': 'num_url'})\ntrain_set = train_set.fillna(0)\n\ntest_set = test_df.merge(bidder_unique, on='bidder_id', how='left').drop(columns = ['payment_account', 'address'], errors = 'ignore')\ntest_set = test_set.rename(columns = {'bid_id': 'num_bids', 'auction': 'num_auct', 'merchandise': 'num_merch_type', 'device': 'num_device_type', 'time': 'num_time', 'country': 'num_ctry', 'ip': 'num_ip', 'url': 'num_url'})\ntest_set = test_set.fillna(0)\n\ntrain_set.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Features generated from first-differencing using .diff()","metadata":{}},{"cell_type":"markdown","source":"The competition mentioned that the relative order and scales of time are preserved. Hence, it can be useful for comparisons between behavior using time.","metadata":{}},{"cell_type":"code","source":"time_df = bids_df.sort_values(['bidder_id', 'time'])\ntime_df.head(10)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can generate features by looking at the time difference between every two bids for each bidder. We cannot take the first difference of each bidder's first bid so there are missing values. As these missing values do not have any intepretations, we drop them.","metadata":{}},{"cell_type":"code","source":"firstdiff = time_df.groupby('bidder_id')[['time']].diff()\nfirstdiff.head(10)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_df['first_diff'] = firstdiff\nfirstdiff_feat = time_df[['bidder_id', 'first_diff']].dropna()\nfirstdiff_feat.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can generate some features using common aggregate functions such as mean, median, minimum and maximum.","metadata":{}},{"cell_type":"code","source":"bid_intervals = firstdiff_feat.groupby('bidder_id')[['first_diff']].describe().reset_index()\nbid_intervals = bid_intervals.droplevel(axis=1, level=0)\nbid_intervals = bid_intervals.rename(columns = {'': 'bidder_id', 'mean': 'mean_diff', 'std': 'std_diff', '50%': 'median_diff', 'min': 'min_diff', 'max': 'max_diff'}).fillna(0)\nbid_intervals['iqr_diff'] = bid_intervals['75%'] - bid_intervals['25%']\nbid_intervals = bid_intervals.drop(['25%', '75%', 'count'], axis = 1)\nbid_intervals.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"When adding the above features to our training and test datasets, we fill missing values for the concurrent bids with 0 simply because the bidders do not have any.","metadata":{}},{"cell_type":"markdown","source":"For the first-difference features, we fill the missing values with the median values because these bidders do not have any bid information.","metadata":{}},{"cell_type":"code","source":"train_set = train_set.merge(bid_intervals, on='bidder_id', how='left')\ntrain_set = train_set.fillna(train_set.median())\n\ntest_set = test_set.merge(bid_intervals, on='bidder_id', how='left')\ntest_set = test_set.fillna(test_set.median())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One interesting observation is that some bidders have zero lag time between two bids. Intuitively, bots may be able to achieve this more than an average human, so we can look at the values where the time difference equals 0. We label such bids as concurrent bids.","metadata":{}},{"cell_type":"code","source":"concurrent_bids = firstdiff_feat[firstdiff_feat['first_diff'] == 0].groupby('bidder_id').count().reset_index()\nconcurrent_bids = concurrent_bids.rename(columns = {'first_diff': 'num_concurrent_bids'})\nconcurrent_bids.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.merge(concurrent_bids, on='bidder_id', how='left').fillna(0)\n\ntest_set = test_set.merge(concurrent_bids, on='bidder_id', how='left').fillna(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Features generated from time using .first()","metadata":{}},{"cell_type":"markdown","source":"Time can be used to indicate the behavior of a bidder relative to the others. The easiest comparison is the number of times a bidder is the first or last in an auction. Intuitively, we fill missing values with 0.","metadata":{}},{"cell_type":"code","source":"first_bid = bids_df.sort_values(['auction', 'time'])\nfirst_bid = first_bid.groupby('auction').first().reset_index()\nfirst_bid = first_bid.groupby('bidder_id').count()['bid_id'].reset_index()\nfirst_bid = first_bid.rename(columns = {'bid_id': 'num_first_bid'})\nfirst_bid.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_bid = bids_df.sort_values(['auction', 'time'], ascending = [True, False])\nlast_bid = last_bid.groupby('auction').first().reset_index()\nlast_bid = last_bid.groupby('bidder_id').count()['bid_id'].reset_index()\nlast_bid = last_bid.rename(columns = {'bid_id': 'num_last_bid'})\nlast_bid.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.merge(first_bid, on='bidder_id', how='left').fillna(0)\ntrain_set = train_set.merge(last_bid, on='bidder_id', how='left').fillna(0)\n\ntest_set = test_set.merge(first_bid, on='bidder_id', how='left').fillna(0)\ntest_set = test_set.merge(last_bid, on='bidder_id', how='left').fillna(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Features by other hypotheses","metadata":{}},{"cell_type":"markdown","source":"##### Ratio of bids in first half to second half of auction","metadata":{}},{"cell_type":"markdown","source":"We define the duration of the auction to be the difference between the first and last bid of that auction. We assume that the bots will bid more towards the end of an auction because bidding early does not really ensure that the bots would not be outbidded.","metadata":{}},{"cell_type":"code","source":"auct_duration = bids_df.sort_values(['auction', 'time'])[['bidder_id', 'auction', 'time']]\nauct_duration = auct_duration[['auction','time']].groupby('auction').agg([max,min]).reset_index().droplevel(axis=1, level=0).rename(columns = {'': 'auction'})\nauct_duration['auct_duration'] = auct_duration['max'] - auct_duration['min']\nauct_duration.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_ratio = bids_df.sort_values(['auction', 'time'])[['bidder_id', 'auction', 'time']]\ntime_ratio = time_ratio.merge(auct_duration, on = 'auction', how = 'left')\ntime_ratio.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_ratio['temp'] = time_ratio['time'] - time_ratio['auct_duration'] / 2\ntime_ratio.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_ratio['firsthalf'] = time_ratio['temp'] < time_ratio['min']\ntime_ratio.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ratio_firsthalf = time_ratio[['bidder_id', 'firsthalf']].groupby('bidder_id').agg(['count', sum]).reset_index().droplevel(axis=1, level=0).rename(columns = {'': 'bidder_id', 'count': 'num_total_bids', 'sum': 'num_firsthalf_bids'})\nratio_firsthalf['num_secondhalf_bids'] = ratio_firsthalf['num_total_bids'] - ratio_firsthalf['num_firsthalf_bids']\nratio_firsthalf['percent_firsthalf_bids'] = ratio_firsthalf['num_firsthalf_bids'] / ratio_firsthalf['num_total_bids']\nratio_firsthalf['percent_secondhalf_bids'] = ratio_firsthalf['num_secondhalf_bids'] / ratio_firsthalf['num_total_bids']\nratio_firsthalf = ratio_firsthalf.drop('num_total_bids', axis = 1)\nratio_firsthalf.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.merge(ratio_firsthalf, on='bidder_id', how='left').fillna(0)\n\ntest_set = test_set.merge(ratio_firsthalf, on='bidder_id', how='left').fillna(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Max number of bids in an auction","metadata":{}},{"cell_type":"markdown","source":"Since the bots' aim is to win the auction, by assuming that a bot will not give up an auction, the bot should realistically make more bids for any auction.","metadata":{}},{"cell_type":"code","source":"max_bids_in_auct = bids_df.groupby(['bidder_id', 'auction']).count().reset_index()[['bidder_id', 'auction', 'bid_id']].rename(columns = {'bid_id': 'max_bids_in_auct'})\nmax_bids_in_auct = max_bids_in_auct[['bidder_id', 'max_bids_in_auct']].groupby('bidder_id').max().reset_index()\nmax_bids_in_auct.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.merge(max_bids_in_auct, on='bidder_id', how='left').fillna(0)\n\ntest_set = test_set.merge(max_bids_in_auct, on='bidder_id', how='left').fillna(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Max number of bids in across devices","metadata":{}},{"cell_type":"markdown","source":"We find out the maximum number of bids made using the same device by a bidder and find out whether humans and bots have different behavior when it comes to switching devices.","metadata":{}},{"cell_type":"code","source":"device = bids_df.groupby(['bidder_id', 'device']).nunique()[['bid_id', 'auction']].reset_index().rename(columns = {'bid_id': 'max_bids_per_device', 'auction': 'num_auct_per_device'})\ndevice = device.groupby('bidder_id').max().reset_index()\ndevice['max_bids_per_device_per_auct'] = device['max_bids_per_device'] / device['num_auct_per_device']\ndevice = device[['bidder_id', 'max_bids_per_device', 'max_bids_per_device_per_auct']]\ndevice.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.merge(device, on='bidder_id', how='left').fillna(0)\n\ntest_set = test_set.merge(device, on='bidder_id', how='left').fillna(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature transformations","metadata":{}},{"cell_type":"markdown","source":"We attempt to create some features that makes sense by intuition. For example, bids_per_auct may be a better feature than just num_bids and num_auct separately because we standardize the number of bids made by a bidder with respect to the total number of auctions they participated. The same can be said for the other features.\n    \nAgain, we fill missing values with 0 because they indicate those bidders with no bid information.","metadata":{}},{"cell_type":"code","source":"train_set['percent_concurrent_bids'] = train_set['num_concurrent_bids'] / train_set['num_bids']\ntrain_set['bids_per_auct'] = train_set['num_bids'] / train_set['num_auct']\ntrain_set['bids_per_device'] = train_set['num_bids'] / train_set['num_device_type']\ntrain_set['bids_per_url'] = train_set['num_bids'] / train_set['num_url']\ntrain_set['device_per_auct'] = train_set['num_device_type'] / train_set['num_auct']\ntrain_set['ip_per_ctry'] = train_set['num_ip'] / train_set['num_ctry']\ntrain_set['percent_max_bids'] = train_set['max_bids_per_device'] / train_set['num_bids']\n\ntrain_set = train_set.fillna(0)\n\ntest_set['percent_concurrent_bids'] = test_set['num_concurrent_bids'] / test_set['num_bids']\ntest_set['bids_per_auct'] = test_set['num_bids'] / test_set['num_auct']\ntest_set['bids_per_device'] = test_set['num_bids'] / test_set['num_device_type']\ntest_set['bids_per_url'] = test_set['num_bids'] / test_set['num_url']\ntest_set['device_per_auct'] = test_set['num_device_type'] / test_set['num_auct']\ntest_set['ip_per_ctry'] = test_set['num_ip'] / test_set['num_ctry']\ntest_set['percent_max_bids'] = test_set['max_bids_per_device'] / test_set['num_bids']\n\ntest_set = test_set.fillna(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Outliers","metadata":{}},{"cell_type":"markdown","source":"There are outliers, with 5 bots having only a single bid. We remove them as they may affect model performance badly.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots()\nnum_bids_per_bot = sns.barplot(data = train_set[train_set['outcome'] == 1].sort_values('num_bids').head(10),\n                               x = 'bidder_id',\n                               y = 'num_bids',\n                               ax = ax\n                              )\nax.bar_label(ax.containers[0])\nplt.xticks(rotation = 90)\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set[train_set['outcome'] == 1].sort_values('num_bids').head(6)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.drop([615, 775, 392, 1669, 1102], axis = 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Checking the significance of features","metadata":{}},{"cell_type":"markdown","source":"We compute the mean of each feature for a human and a bot. One interesting observation is that both humans and bots only bidded for one type of merchandise. The value for humans is skewed due to humans with no bid data.\n\nWe can investigate the feature, merchandise, further to see if encoding can be performed.","metadata":{}},{"cell_type":"code","source":"train_set.groupby('outcome').mean().T","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The top 3 merchandises bidded by both humans and robots are the same, namely sporting goods, mobile and jewelry. Some merchandise like home goods and autoparts are not bidded by robots. However, they make up a small proportion of the human bids. We choose not to include categorical variables in our model.","metadata":{}},{"cell_type":"code","source":"temp_df = bids_df.merge(train_df, on = 'bidder_id', how = 'left').dropna()\ntemp_df.groupby(['outcome', 'merchandise']).count()[['bid_id']].sort_values(['outcome', 'bid_id'], ascending = False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.drop('num_merch_type', axis = 1)\n\ntest_set = test_set.drop('num_merch_type', axis = 1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_features = train_set.columns.drop(['bidder_id', 'outcome'])\nall_features","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Another way to check the significance of the features is to use a density plot. From the plots below, the features num_url, num_concurrent_bids, num_firsthalf_bids, max_bids_per_device have similar distributions for both human and bot. Hence, we choose to exclude them from our model as they may not help to differentiate between a human and a bot well.","metadata":{}},{"cell_type":"code","source":"nrows = 15\nncols = 2\nfig, axes = plt.subplots(nrows = nrows, ncols = ncols, figsize = (18, 60))\nfor i, column in enumerate(all_features):\n    humans = sns.kdeplot(data = train_set[train_set['outcome'] == 0],\n                x = column,\n                ax = axes[i // ncols, i % ncols],\n                color = 'blue',\n                fill = True,\n                alpha = 0.1,\n                linewidth = 2,\n                label = 'Human').set_xlim(left = 0)\n    bots = sns.kdeplot(data = train_set[train_set['outcome'] == 1],\n                x = column,\n                ax = axes[i // ncols, i % ncols],\n                color = 'red',\n                fill = True,\n                alpha = 0.1,\n                linewidth = 2,\n                label = 'Bot').set_xlim(left = 0)\n    axes[i // ncols,i % ncols].legend()\n    \nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Validation","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import GridSearchCV, RepeatedStratifiedKFold\nfrom time import time\nfrom imblearn.over_sampling import RandomOverSampler\nfrom imblearn.pipeline import make_pipeline","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imbalanced data","metadata":{}},{"cell_type":"code","source":"train_set['outcome'].value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Due to imbalanced data, we perform over-sampling using RandomOverSampler.\n\nAfterwards, we choose to do ensemble averaging of multiple Random Forest models to reduce the variance of our predictions. We do that by setting different random_state for each model. We then perform hyperparameter tuning for each model separately using GridSearchCV.","metadata":{}},{"cell_type":"markdown","source":"### Initialize training/test data and models","metadata":{}},{"cell_type":"code","source":"feature_col = train_set.columns.drop(['bidder_id', 'outcome',\n                                     'num_url',  'num_concurrent_bids',  'num_firsthalf_bids', 'max_bids_per_device'\n                                     ])\nprint(feature_col)\n\nX = train_set[feature_col]\ny = train_set['outcome']\n\nX_kaggle = test_set[feature_col]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We first initialize the base models and find out the AUC as reference.","metadata":{}},{"cell_type":"code","source":"rf1 = RandomForestClassifier(random_state = 0)\nrf2 = RandomForestClassifier(random_state = 123)\nrf3 = RandomForestClassifier(random_state = 456)\nrf4 = RandomForestClassifier(random_state = 789)\nrf5 = RandomForestClassifier(random_state = 999)\n\nros = RandomOverSampler(sampling_strategy = 0.1, random_state = 456)\n\npp1 = make_pipeline(ros, rf1)\npp2 = make_pipeline(ros, rf2)\npp3 = make_pipeline(ros, rf3)\npp4 = make_pipeline(ros, rf4)\npp5 = make_pipeline(ros, rf5)\n\nbase_models = [pp1, pp2, pp3, pp4, pp5]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cv(models, X, y):\n    start = time()\n\n    rskfold = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 3, random_state = 456) \n    k_fold_AUC = []\n\n    for train_index, test_index in rskfold.split(X, y):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n\n        y_proba = []\n        for model in models:\n            model.fit(X_train, y_train)\n            sub_y_proba = model.predict_proba(X_test)[:, 1]\n            y_proba.append(sub_y_proba)\n        y_proba = np.mean(y_proba, axis = 0)\n\n        AUC = roc_auc_score(y_test, y_proba)\n        k_fold_AUC.append(AUC)\n    \n    mean_AUC = np.mean(k_fold_AUC)\n    \n    end = time()\n\n    print(f\"Time elapsed: {(end - start):.4f} seconds\")\n    print(f\"AUC:{mean_AUC:.4f}\")\n    \n    return k_fold_AUC","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### One individual model before hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"base_individual_AUCs = cv([base_models[2]], X, y) # model with random_state = 456 used as reference","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensemble averaging before hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"base_average_AUCs = cv(base_models, X, y)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hyperparameter tuning with GridSearchCV","metadata":{}},{"cell_type":"code","source":"grid = {'randomforestclassifier__n_estimators': [100, 200, 300],\n        'randomforestclassifier__max_depth': [None, 5, 8, 10],\n        'randomforestclassifier__min_samples_split': [2, 5, 10],\n        'randomforestclassifier__min_samples_leaf': [1, 2, 4],\n       }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time()\n\nbest_models = []\n\nfor model in base_models:\n\n    search = GridSearchCV(estimator = model,\n                          param_grid = grid,\n                          scoring = 'roc_auc', \n                          cv = 3, # default is StratifiedKFold as estimator is classifier\n                          verbose = 2,\n                          n_jobs = -1)\n    search.fit(X,y)\n    best_models.append(search.best_estimator_)\n\nend = time()\n\nprint(f\"Time Elapsed: {(end - start):.4f} seconds\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(best_models)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### One individual model after hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"individual_AUCs = cv([best_models[2]], X, y) # again, model with random_state = 456 used as reference","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensemble averaging after hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"average_AUCs =cv(best_models, X, y)","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Summary","metadata":{}},{"cell_type":"markdown","source":"We plot the distribution of the ROC AUC scores to compare the differences from ensemble averaging and hyperparameter tuning.","metadata":{}},{"cell_type":"code","source":"model_AUCs_list = [('Base individual RF', base_individual_AUCs), ('Base average RFs', base_average_AUCs),\n                   ('Tuned individual RF', individual_AUCs), ('Tuned average RFs', average_AUCs)]\n\nfig, ax = plt.subplots(figsize = (18, 6))\n\nfor i in range(len(model_AUCs_list)):\n    color = next(ax._get_lines.prop_cycler)['color']\n    \n    AUC_distribution = sns.kdeplot(x = model_AUCs_list[i][1],\n                                   ax = ax,\n                                   label = model_AUCs_list[i][0],\n                                   color = color\n                                  )\n    \n    x_coord = ax.lines[-1].get_xdata()\n    y_coord = ax.lines[-1].get_ydata()\n    index_of_max_y = np.argmax(y_coord)\n    ax.axvline(x_coord[index_of_max_y], linestyle = '--', linewidth = 1, color = color)\n\nax.legend(loc = 'upper left', fontsize = 'x-large')\nplt.title('ROC AUC score distribution across 4 models', fontsize = 'xx-large')\nplt.xlabel('ROC AUC score', fontsize = 'x-large')\nplt.ylabel('Density', fontsize = 'x-large')\nplt.xlim(right = 1)\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is evident that hyperparameter tuning reduced the variance of ROC AUC scores as indicated by the spread of the green and red curves.\n\nAlso, the ROC AUC score for tuned ensemble average of random forests (red vertical line) is more likely to be higher compared to the 3 other models, as seen by the vertical line indicators of their respective peaks.","metadata":{}},{"cell_type":"markdown","source":"# Final Model","metadata":{}},{"cell_type":"code","source":"final_models = best_models","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"proba = []\n\nfor model in final_models:\n    model.fit(X, y)\n\n    proba_rforest = model.predict_proba(X_kaggle)[:,1]\n    proba.append(proba_rforest)\n\nresult = np.mean(proba, axis = 0)\nresult","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_dataframe = pd.DataFrame({\n    'bidder_id': test_set['bidder_id'],\n    'prediction': result\n})\noutput_dataframe.to_csv('my_predictions.csv', index=False) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}}]}