{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-15T22:39:04.647959Z","iopub.execute_input":"2022-01-15T22:39:04.648231Z","iopub.status.idle":"2022-01-15T22:39:04.656804Z","shell.execute_reply.started":"2022-01-15T22:39:04.648200Z","shell.execute_reply":"2022-01-15T22:39:04.656036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**1.0 Data loading**\n\nExtract zip files and load data into panda dataframe.","metadata":{}},{"cell_type":"code","source":"# Extrac zip files\nimport zipfile\ndef extract_images(filePath):\n    with zipfile.ZipFile(filePath,\"r\") as z:\n        z.extractall(\".\")\n# Test\nextract_images('/kaggle/input/facebook-recruiting-iv-human-or-bot/train.csv.zip')\nextract_images('/kaggle/input/facebook-recruiting-iv-human-or-bot/bids.csv.zip')\nextract_images('/kaggle/input/facebook-recruiting-iv-human-or-bot/test.csv.zip')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:04.658624Z","iopub.execute_input":"2022-01-15T22:39:04.659278Z","iopub.status.idle":"2022-01-15T22:39:11.430937Z","shell.execute_reply.started":"2022-01-15T22:39:04.659244Z","shell.execute_reply":"2022-01-15T22:39:11.430068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/working/train.csv')\nbids_df = pd.read_csv('/kaggle/working/bids.csv')\ntest_df = pd.read_csv('/kaggle/working/test.csv')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:11.432781Z","iopub.execute_input":"2022-01-15T22:39:11.434445Z","iopub.status.idle":"2022-01-15T22:39:23.914725Z","shell.execute_reply.started":"2022-01-15T22:39:11.434401Z","shell.execute_reply":"2022-01-15T22:39:23.914010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:23.915808Z","iopub.execute_input":"2022-01-15T22:39:23.916050Z","iopub.status.idle":"2022-01-15T22:39:23.930660Z","shell.execute_reply.started":"2022-01-15T22:39:23.916017Z","shell.execute_reply":"2022-01-15T22:39:23.929761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:23.933149Z","iopub.execute_input":"2022-01-15T22:39:23.933534Z","iopub.status.idle":"2022-01-15T22:39:23.945721Z","shell.execute_reply.started":"2022-01-15T22:39:23.933433Z","shell.execute_reply":"2022-01-15T22:39:23.945016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.0 Analyze Dataset**\nThe datasets are provided by two part:\n* train.csv: the base bidder dataset with labels. Besides label, it also contains unique bidder_id followed with payment_account, address and outcome;\n* bids.csv: the dataset contains some more useful information associated with unique key bid_id, corresponding with bidder_id, auction, merchandise, device, time, country, ip, and url. time and url are encrypted information.\n\nSince both of these two files contains the unique key bidder_id, we firstly joined these data by this key. Our object is to try to find each bid is made by a human or robot, thus, we only consider the data which have bids information and labelled. We will left join on train.csv and check the joined data for missing bids information.","metadata":{}},{"cell_type":"code","source":"# Populate train and test data with bid details\ndef populate_bids(df):\n    df = pd.merge(left=df, right=bids_df, how='left', left_on='bidder_id', right_on='bidder_id')\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:23.947112Z","iopub.execute_input":"2022-01-15T22:39:23.947521Z","iopub.status.idle":"2022-01-15T22:39:23.953764Z","shell.execute_reply.started":"2022-01-15T22:39:23.947349Z","shell.execute_reply":"2022-01-15T22:39:23.952930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = populate_bids(test_df)\ntest.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:23.954901Z","iopub.execute_input":"2022-01-15T22:39:23.955442Z","iopub.status.idle":"2022-01-15T22:39:37.303909Z","shell.execute_reply.started":"2022-01-15T22:39:23.955400Z","shell.execute_reply":"2022-01-15T22:39:37.303230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = populate_bids(train_df)\ntrain.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:37.305130Z","iopub.execute_input":"2022-01-15T22:39:37.305439Z","iopub.status.idle":"2022-01-15T22:39:46.222888Z","shell.execute_reply.started":"2022-01-15T22:39:37.305410Z","shell.execute_reply":"2022-01-15T22:39:46.222193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Cleaning**\nSince we can see all the missing values are from the variables that come from bids, we want to check if there exist some of bidder_id do not have any bids.","metadata":{}},{"cell_type":"code","source":"nobids_train = train[pd.isnull(train['bid_id'])]['bidder_id'].unique()\nlen(nobids_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:46.224200Z","iopub.execute_input":"2022-01-15T22:39:46.224605Z","iopub.status.idle":"2022-01-15T22:39:47.260411Z","shell.execute_reply.started":"2022-01-15T22:39:46.224568Z","shell.execute_reply":"2022-01-15T22:39:47.259736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nobids_test = test[pd.isnull(test['bid_id'])]['bidder_id'].unique()\nlen(nobids_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:47.261749Z","iopub.execute_input":"2022-01-15T22:39:47.262033Z","iopub.status.idle":"2022-01-15T22:39:48.844008Z","shell.execute_reply.started":"2022-01-15T22:39:47.261998Z","shell.execute_reply":"2022-01-15T22:39:48.843308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check outcome for these in the train\ntrain[train['bidder_id'].isin(nobids_train)]['outcome']","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:48.845382Z","iopub.execute_input":"2022-01-15T22:39:48.845637Z","iopub.status.idle":"2022-01-15T22:39:48.956255Z","shell.execute_reply.started":"2022-01-15T22:39:48.845604Z","shell.execute_reply":"2022-01-15T22:39:48.955347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There exist 29 bidder in training data set having no bid data. After checking with their labels, they were all marked as human. Since we aim to detect robot, we will just ignore them. We will update `populate_bids` function.","metadata":{}},{"cell_type":"code","source":"# Populate train and test data with bid details\ndef populate_bids_with_data_cleanup(df):\n    df = pd.merge(left=df, right=bids_df, how='left', left_on='bidder_id', right_on='bidder_id')\n    nobids = df[pd.isnull(df['bid_id'])]['bidder_id'].unique()\n    # for all the missing value observation, drop it\n    df = df[~df['bidder_id'].isin(nobids)]\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:48.958303Z","iopub.execute_input":"2022-01-15T22:39:48.958832Z","iopub.status.idle":"2022-01-15T22:39:48.966329Z","shell.execute_reply.started":"2022-01-15T22:39:48.958795Z","shell.execute_reply":"2022-01-15T22:39:48.965688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = populate_bids_with_data_cleanup(test_df)\ntest_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:39:48.971247Z","iopub.execute_input":"2022-01-15T22:39:48.971475Z","iopub.status.idle":"2022-01-15T22:40:04.599209Z","shell.execute_reply.started":"2022-01-15T22:39:48.971441Z","shell.execute_reply":"2022-01-15T22:40:04.598564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = populate_bids_with_data_cleanup(train_df)\ntrain_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:04.600596Z","iopub.execute_input":"2022-01-15T22:40:04.600882Z","iopub.status.idle":"2022-01-15T22:40:15.122516Z","shell.execute_reply.started":"2022-01-15T22:40:04.600848Z","shell.execute_reply":"2022-01-15T22:40:15.121770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the missing value ratio of the country entry in train data\ncount_miss_ratio_train = pd.isnull(train_data['country']).sum()/len(train_data)*100\nprint(f\" %0.2f %% observations missing country entry in train data.\" %\n      float(count_miss_ratio_train))","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:15.123865Z","iopub.execute_input":"2022-01-15T22:40:15.124162Z","iopub.status.idle":"2022-01-15T22:40:15.412649Z","shell.execute_reply.started":"2022-01-15T22:40:15.124128Z","shell.execute_reply":"2022-01-15T22:40:15.411828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the missing value ratio of the country entry in test data\ncount_miss_ratio_test = pd.isnull(test_data['country']).sum()/len(test_data)*100\nprint(f\" %0.2f %% observations missing country entry in test data.\" %\n      float(count_miss_ratio_test))","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:15.414197Z","iopub.execute_input":"2022-01-15T22:40:15.414442Z","iopub.status.idle":"2022-01-15T22:40:15.840617Z","shell.execute_reply.started":"2022-01-15T22:40:15.414389Z","shell.execute_reply":"2022-01-15T22:40:15.839744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"For the full data set, there is only few of observations missing country entry. We will just ignore the missing country entry data.","metadata":{}},{"cell_type":"code","source":"# Populate train and test data with bid details\ndef populate_bids_with_data_cleanup_final(df):\n    df = pd.merge(left=df, right=bids_df, how='left', left_on='bidder_id', right_on='bidder_id')\n    nobids = df[pd.isnull(df['bid_id'])]['bidder_id'].unique()\n    # for all the missing value observation, drop it\n    df = df[~df['bidder_id'].isin(nobids)]\n    # remove missing country data\n    df = df.dropna()\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:15.842094Z","iopub.execute_input":"2022-01-15T22:40:15.842352Z","iopub.status.idle":"2022-01-15T22:40:15.849949Z","shell.execute_reply.started":"2022-01-15T22:40:15.842318Z","shell.execute_reply":"2022-01-15T22:40:15.849271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = populate_bids_with_data_cleanup_final(train_df)\ntrain.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:15.851246Z","iopub.execute_input":"2022-01-15T22:40:15.851672Z","iopub.status.idle":"2022-01-15T22:40:30.123904Z","shell.execute_reply.started":"2022-01-15T22:40:15.851635Z","shell.execute_reply":"2022-01-15T22:40:30.121277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = populate_bids_with_data_cleanup_final(test_df)\ntest.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:30.124853Z","iopub.execute_input":"2022-01-15T22:40:30.125079Z","iopub.status.idle":"2022-01-15T22:40:51.226626Z","shell.execute_reply.started":"2022-01-15T22:40:30.125041Z","shell.execute_reply":"2022-01-15T22:40:51.225908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3.0 Data Exploration**","metadata":{}},{"cell_type":"code","source":"# check the unique number of each feature in train\nprint(f\"total row in train:          {len(train)}\")\nprint(f\"total bids in train:         {len(train['bid_id'].unique())}\")\nprint(f\"total bidder in train:      {len(train['bidder_id'].unique())}\")\nprint(f\"total payment in train:     {len(train['payment_account'].unique())}\")\nprint(f\"total address in train:     {len(train['address'].unique())}\")\nprint(f\"total auction in train:     {len(train['auction'].unique())}\")\nprint(f\"total merchandise in train: {len(train['merchandise'].unique())}\")\nprint(f\"total device in train:      {len(train['device'].unique())}\")\nprint(f\"total country in train:     {len(train['country'].unique())}\")\nprint(f\"total ip in train:          {len(train['ip'].unique())}\")\nprint(f\"total url in train:         {len(train['url'].unique())}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:51.227810Z","iopub.execute_input":"2022-01-15T22:40:51.228055Z","iopub.status.idle":"2022-01-15T22:40:54.624071Z","shell.execute_reply.started":"2022-01-15T22:40:51.228020Z","shell.execute_reply":"2022-01-15T22:40:54.623386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From this output, we can see the payment and address are one-to-one correspondence to bidder. Since our outpue, whether a bid is made from a human or robot, is label based on bidder, we can igonre these two features for future analysis. Since merchandise and country are compared low in this situation, we can just treat themselves as a feature for analysis.\n\nFor the other features, let's check the ratios to robots and human. Before that, let's check the robots vs. human first.","metadata":{}},{"cell_type":"code","source":"print(\n    f\"total bids made by robots in data set: {len(train[train['outcome'] == 1])}\")\nprint(\n    f\"total bids made by human in data set:  {len(train[train['outcome'] == 0])}\")\nprint(\"the ratio of made by robot vs. bids made by human in data set: 3:20\")","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:54.625202Z","iopub.execute_input":"2022-01-15T22:40:54.625898Z","iopub.status.idle":"2022-01-15T22:40:55.148781Z","shell.execute_reply.started":"2022-01-15T22:40:54.625856Z","shell.execute_reply":"2022-01-15T22:40:55.148039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These trainning data set has unbalanced number of robots and human:\n\nWe might need to do resampling before training the model.\nfor the evaluation metrices, we will foucus on AUC and precision vs. recall.\nThen checking the features' ratio of robots and human:","metadata":{}},{"cell_type":"code","source":"def get_metrics(bid):\n    print(f\"average bids per robot: %.0f\" % float(len(\n        bid[bid['outcome'] == 1])/len(bid[bid['outcome'] == 1]['bidder_id'].unique())))\n    print(f\"average bids per human:  %.0f\" % float(len(\n        bid[bid['outcome'] == 0])/len(bid[bid['outcome'] == 0]['bidder_id'].unique())))\n    print(f\"average bids per auction by robot: %.0f\" % float(\n        len(bid[bid['outcome'] == 1])/len(bid[bid['outcome'] == 1]['auction'].unique())))\n    print(f\"average bids per auction by human:  %.0f\" % float(\n        len(bid[bid['outcome'] == 0])/len(bid[bid['outcome'] == 0]['auction'].unique())))\n    print(f\"average bids per device by robot: %.0f\" % float(\n        len(bid[bid['outcome'] == 1])/len(bid[bid['outcome'] == 1]['device'].unique())))\n    print(f\"average bids per device by human:  %.0f\" % float(\n        len(bid[bid['outcome'] == 0])/len(bid[bid['outcome'] == 0]['device'].unique())))\n    print(f\"average bids per ip by robots: %.2f\" % float(\n        len(bid[bid['outcome'] == 1])/len(bid[bid['outcome'] == 1]['ip'].unique())))\n    print(f\"average bids per ip by human:  %.2f\" % float(\n        len(bid[bid['outcome'] == 0])/len(bid[bid['outcome'] == 0]['ip'].unique())))\n    print(f\"average bids per url by robot: %.2f\" % float(\n        len(bid[bid['outcome'] == 1])/len(bid[bid['outcome'] == 1]['url'].unique())))\n    print(f\"average bids per url by human:  %.2f\" % float(\n        len(bid[bid['outcome'] == 0])/len(bid[bid['outcome'] == 0]['url'].unique())))\n\nget_metrics(train)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:40:55.150008Z","iopub.execute_input":"2022-01-15T22:40:55.150349Z","iopub.status.idle":"2022-01-15T22:41:02.846834Z","shell.execute_reply.started":"2022-01-15T22:40:55.150309Z","shell.execute_reply":"2022-01-15T22:41:02.846063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above, we can see distinct difference between bids made by human and bots per auction, per device. Features related to number of bids, auction, device draw our attention.","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"**4. Feature Engineering**","metadata":{}},{"cell_type":"code","source":"# bidding time difference per user (bidder_id)\ndef calculate_bidding_time_difference_per_user(df):\n    df = df.sort_values(by=['time'])\n    df['timediffs'] = df.groupby('bidder_id')['time'].transform(pd.Series.diff)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:02.847950Z","iopub.execute_input":"2022-01-15T22:41:02.848196Z","iopub.status.idle":"2022-01-15T22:41:02.854265Z","shell.execute_reply.started":"2022-01-15T22:41:02.848159Z","shell.execute_reply":"2022-01-15T22:41:02.853272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# number of bids a user made per auction\ndef bids_user_made_per_auction(df):\n    bids_per_auction = df.groupby(['auction', 'bidder_id']).size()\n    return bids_per_auction.to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:02.855590Z","iopub.execute_input":"2022-01-15T22:41:02.855991Z","iopub.status.idle":"2022-01-15T22:41:02.864570Z","shell.execute_reply.started":"2022-01-15T22:41:02.855956Z","shell.execute_reply":"2022-01-15T22:41:02.863872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# proportion of bots for each country\ndef proportion_of_bots_for_each_country(df):\n    pbots_country = df[df['outcome'] == 1].groupby('country').size()/df.groupby('country').size()\n    pbots_country = pbots_country.fillna(0)\n    return pbots_country.to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:02.866206Z","iopub.execute_input":"2022-01-15T22:41:02.866433Z","iopub.status.idle":"2022-01-15T22:41:02.874628Z","shell.execute_reply.started":"2022-01-15T22:41:02.866409Z","shell.execute_reply":"2022-01-15T22:41:02.873887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# proportion of bots per device\ndef proportion_of_bots_per_device(df):\n    pbots_device = df[df['outcome'] == 1].groupby('device').size()/df.groupby('device').size()\n    pbots_device = pbots_device.fillna(0)\n    return pbots_device.to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:02.875805Z","iopub.execute_input":"2022-01-15T22:41:02.876199Z","iopub.status.idle":"2022-01-15T22:41:02.886644Z","shell.execute_reply.started":"2022-01-15T22:41:02.876164Z","shell.execute_reply":"2022-01-15T22:41:02.885921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# number of unique ip to number of bids ratio\ndef number_of_unique_ip_to_number_of_bids_ratio(df):\n    ip_bids_ratio = df.groupby('bidder_id')['ip'].nunique()/df.groupby('bidder_id')['bid_id'].nunique()\n    return ip_bids_ratio.to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:02.887901Z","iopub.execute_input":"2022-01-15T22:41:02.888256Z","iopub.status.idle":"2022-01-15T22:41:02.894995Z","shell.execute_reply.started":"2022-01-15T22:41:02.888222Z","shell.execute_reply":"2022-01-15T22:41:02.894338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import stats\n\ndef ent(data):\n    \"\"\"\n    Calculate the entropy\n\n    Parameters\n    ----------\n    data : dataframe\n        a DataFrame containing original data\n\n    Returns\n    -------\n    output: float\n        computed entropy \n\n    Notes\n    -----\n    entropy is calculated with the following steps:\n        1. compute the probabilities\n        2. fit in the entropy formula\n    \"\"\"\n    p_data = data.value_counts()/len(data)  # calculates the probabilities\n    # input probabilities to get the entropy\n    entropy = stats.entropy(p_data)\n    return entropy","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:02.896333Z","iopub.execute_input":"2022-01-15T22:41:02.896626Z","iopub.status.idle":"2022-01-15T22:41:03.598665Z","shell.execute_reply.started":"2022-01-15T22:41:02.896592Z","shell.execute_reply":"2022-01-15T22:41:03.597925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mean per auction url entropy for each user\n# Input a pandas series\ndef mean_per_auctio_url_entropy_for_each_user(df):\n    auction_url_entropy = df.groupby(['auction', 'bidder_id'])['url'].apply(ent)\n    return auction_url_entropy.groupby('bidder_id').mean().reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:03.599978Z","iopub.execute_input":"2022-01-15T22:41:03.600227Z","iopub.status.idle":"2022-01-15T22:41:03.604846Z","shell.execute_reply.started":"2022-01-15T22:41:03.600195Z","shell.execute_reply":"2022-01-15T22:41:03.603939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_engineering(df):\n    df = df.sort_values(by=['time'])\n    df['timediffs'] = df.groupby('bidder_id')['time'].transform(pd.Series.diff)\n    df = pd.merge(df, bids_user_made_per_auction(df), on=['auction', 'bidder_id'], how='left')\n    df = pd.merge(df, proportion_of_bots_for_each_country(df), on='country', how='left')\n    df = pd.merge(df, proportion_of_bots_per_device(df), on='device', how='left')\n    df = pd.merge(df, number_of_unique_ip_to_number_of_bids_ratio(df), on='bidder_id', how='left')\n    df = pd.merge(df, mean_per_auctio_url_entropy_for_each_user(df), on='bidder_id', how='left')\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:03.606243Z","iopub.execute_input":"2022-01-15T22:41:03.606814Z","iopub.status.idle":"2022-01-15T22:41:03.617099Z","shell.execute_reply.started":"2022-01-15T22:41:03.606772Z","shell.execute_reply":"2022-01-15T22:41:03.616406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = feature_engineering(train)\ntrain.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:41:03.618253Z","iopub.execute_input":"2022-01-15T22:41:03.618465Z","iopub.status.idle":"2022-01-15T22:42:23.630962Z","shell.execute_reply.started":"2022-01-15T22:41:03.618442Z","shell.execute_reply":"2022-01-15T22:42:23.630234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:42:23.632320Z","iopub.execute_input":"2022-01-15T22:42:23.632590Z","iopub.status.idle":"2022-01-15T22:42:23.639447Z","shell.execute_reply.started":"2022-01-15T22:42:23.632547Z","shell.execute_reply":"2022-01-15T22:42:23.638539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set column names\ntrain.columns = ['bidder_id', 'payment_account', 'address', 'outcome',\n               'bid_id', 'auction', 'merchandise', 'device', 'time', 'country',\n               'ip', 'url', 'timediffs', 'bids_per_auction', 'pbots_country', 'pbots_device',\n               'ip_bids_ratio', 'auction_url_entropy']\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:42:23.640945Z","iopub.execute_input":"2022-01-15T22:42:23.641233Z","iopub.status.idle":"2022-01-15T22:42:23.668857Z","shell.execute_reply.started":"2022-01-15T22:42:23.641199Z","shell.execute_reply":"2022-01-15T22:42:23.668079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.timediffs = train.timediffs.fillna(0)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:55:27.602112Z","iopub.execute_input":"2022-01-15T22:55:27.602402Z","iopub.status.idle":"2022-01-15T22:55:27.647070Z","shell.execute_reply.started":"2022-01-15T22:55:27.602353Z","shell.execute_reply":"2022-01-15T22:55:27.646341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**5. Data Visualization**\nCheck whether the new features could help us identify human and robots, plot the distribution of each feature of two different type of bidders.","metadata":{}},{"cell_type":"code","source":"!pip install beautifultable","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:42:23.742043Z","iopub.execute_input":"2022-01-15T22:42:23.742468Z","iopub.status.idle":"2022-01-15T22:42:32.699026Z","shell.execute_reply.started":"2022-01-15T22:42:23.742429Z","shell.execute_reply":"2022-01-15T22:42:32.697920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data visualization package\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n# from ggplot import *\nfrom beautifultable import BeautifulTable\n# ignore the warning\nimport warnings","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:42:32.704689Z","iopub.execute_input":"2022-01-15T22:42:32.705397Z","iopub.status.idle":"2022-01-15T22:42:32.836980Z","shell.execute_reply.started":"2022-01-15T22:42:32.705338Z","shell.execute_reply":"2022-01-15T22:42:32.836204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T22:42:32.839310Z","iopub.execute_input":"2022-01-15T22:42:32.839909Z","iopub.status.idle":"2022-01-15T22:42:32.843933Z","shell.execute_reply.started":"2022-01-15T22:42:32.839872Z","shell.execute_reply":"2022-01-15T22:42:32.843276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bots = train.loc[train.outcome == 1]\nhuman = train.loc[train.outcome == 0]\nbots.shape, human.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:02:38.977105Z","iopub.execute_input":"2022-01-15T23:02:38.977377Z","iopub.status.idle":"2022-01-15T23:02:39.560414Z","shell.execute_reply.started":"2022-01-15T23:02:38.977334Z","shell.execute_reply":"2022-01-15T23:02:39.559733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 2, figsize=(14, 12), sharex=False)\nsns.distplot(train['bids_per_auction'], hist=False, kde=True,\n             bins=int(180/5), color='darkblue',\n             kde_kws={'linewidth': 1.5}, ax=axes[0, 0])\nsns.distplot(human['bids_per_auction'], hist=False, kde=True,\n             bins=int(180/5), color='darkred',\n             kde_kws={'linewidth': 1.5}, ax=axes[0, 0])\n\nsns.distplot(bots['pbots_country'], hist=False, kde=True,\n             bins=int(180/5), color='darkblue',\n             kde_kws={'linewidth': 1.5}, ax=axes[0, 1])\nsns.distplot(human['pbots_country'], hist=False, kde=True,\n             bins=int(180/5), color='darkred',\n             kde_kws={'linewidth': 1.5}, ax=axes[0, 1])\n\nsns.distplot(bots['pbots_device'], hist=False, kde=True,\n             bins=int(180/5), color='darkblue', label='bots',\n             kde_kws={'linewidth': 1.5}, ax=axes[1, 0])\nsns.distplot(human['pbots_device'], hist=False, kde=True,\n             bins=int(180/5), color='darkred', label='human',\n             kde_kws={'linewidth': 1.5}, ax=axes[1, 0])\n\nsns.distplot(bots['ip_bids_ratio'], hist=False, kde=True,\n             bins=int(180/5), color='darkblue',\n             kde_kws={'linewidth': 1.5}, ax=axes[1, 1])\nsns.distplot(human['ip_bids_ratio'], hist=False, kde=True,\n             bins=int(180/5), color='darkred',\n             kde_kws={'linewidth': 1.5}, ax=axes[1, 1])\n\nsns.distplot(bots['auction_url_entropy'], hist=False, kde=True,\n             bins=int(180/5), color='darkblue',\n             kde_kws={'linewidth': 1.5}, ax=axes[2, 0])\nsns.distplot(human['auction_url_entropy'], hist=False, kde=True,\n             bins=int(180/5), color='darkred',\n             kde_kws={'linewidth': 1.5}, ax=axes[2, 0])\n\nsns.distplot(bots['timediffs'], hist=False, kde=True,\n             bins=int(180/5), color='darkblue',\n             kde_kws={'linewidth': 1.5}, ax=axes[2, 1])\nsns.distplot(human['timediffs'], hist=False, kde=True,\n             bins=int(180/5), color='darkred',\n             kde_kws={'linewidth': 1.5}, ax=axes[2, 1])\nplt.legend(['bots', 'human'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:03:13.024541Z","iopub.execute_input":"2022-01-15T23:03:13.025153Z","iopub.status.idle":"2022-01-15T23:04:20.262823Z","shell.execute_reply.started":"2022-01-15T23:03:13.025112Z","shell.execute_reply":"2022-01-15T23:04:20.262120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the variable timediff (time differ per user) showed a similar distribution between robots and human. It also has no clearly correlation with our outcome. Therefore, this variable may not be very helpful when we are fitting a model. In contrast, we may even have overfitting problems if we incorporate it in our models. Therefore, we will ignore the variable timediff during the modeling process.\n","metadata":{}},{"cell_type":"code","source":"# Plot the correlation matrix for the numerical values\ncorr_matrix = train.corr()\nsns.heatmap(corr_matrix.corr(),\n            xticklabels=corr_matrix.corr().columns,\n            yticklabels=corr_matrix.corr().columns,\n            cmap=\"Blues\",\n            fmt='d')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:08:21.383710Z","iopub.execute_input":"2022-01-15T23:08:21.383968Z","iopub.status.idle":"2022-01-15T23:08:22.432847Z","shell.execute_reply.started":"2022-01-15T23:08:21.383939Z","shell.execute_reply":"2022-01-15T23:08:22.432195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This plot is the correlation matrix heatmap for our engineered features. Dark blue in this heatmap means two variables are highly positively correlated with each other. The white color in this heatmap indicates that two features are highly negatively correlated with each other. From this plot, we can infer that proportion of bots per device, proportion of bots for each country and number of unique ip to number of bids ratio are highly positively correlated with our outcome. While the mean per auction url entropy for each user are highly negatively correlated with our outcome. However, the variable time differ per user does not show clearly corrrelation with our outcome.\n","metadata":{}},{"cell_type":"markdown","source":"**6. Process imblance data**","metadata":{}},{"cell_type":"code","source":"# for train and test data set split\nfrom sklearn.model_selection import train_test_split\nbid_train, bid_test = train_test_split(train, test_size=0.2)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:10:22.977142Z","iopub.execute_input":"2022-01-15T23:10:22.977684Z","iopub.status.idle":"2022-01-15T23:10:25.973057Z","shell.execute_reply.started":"2022-01-15T23:10:22.977641Z","shell.execute_reply":"2022-01-15T23:10:25.972293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Undersampling**\n\nFrom data preprocessing part, we see the ratio of the robot bidder and the human bidder is 3:20, thus we need to undersampling the human bidder samples in the trainset to keep the data balance.\n\n","metadata":{}},{"cell_type":"code","source":"# Balance train data\nbots_train = bid_train.loc[bid_train.outcome == 1]\nhuman_train = bid_train.loc[bid_train.outcome == 0]\nhuman_sample = human_train.sample(n=len(bots_train))\nbid_train_balance = pd.concat([bots_train, human_sample])\nbid_train_balance.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:14:27.462178Z","iopub.execute_input":"2022-01-15T23:14:27.462455Z","iopub.status.idle":"2022-01-15T23:14:28.639204Z","shell.execute_reply.started":"2022-01-15T23:14:27.462424Z","shell.execute_reply":"2022-01-15T23:14:28.638381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_columns = ['bids_per_auction','pbots_country','pbots_device','ip_bids_ratio','auction_url_entropy']\nX_train = bid_train_balance[train_columns]\ny_train = bid_train_balance['outcome']\n\nX_test = bid_test[train_columns]\ny_test = bid_test['outcome']\nX_train.shape, y_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:22:21.370798Z","iopub.execute_input":"2022-01-15T23:22:21.371341Z","iopub.status.idle":"2022-01-15T23:22:21.396118Z","shell.execute_reply.started":"2022-01-15T23:22:21.371302Z","shell.execute_reply":"2022-01-15T23:22:21.395148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**7. Modeling**","metadata":{}},{"cell_type":"markdown","source":"**First Model : Decision Tree**","metadata":{}},{"cell_type":"code","source":"# for decision tree model\nfrom sklearn import tree\n# for grid search\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RandomizedSearchCV\n\n# hyperparameter tuning\ndt = tree.DecisionTreeClassifier()\nparam_grid = {\n    'criterion': ['gini', 'entropy'],\n    'max_depth': range(3, 6),\n    'max_leaf_nodes': range(10, 15),\n    'min_samples_split': range(2, 6)\n}\n\ndt_cv = GridSearchCV(estimator=dt,\n                     param_grid=param_grid,\n                     cv=5)\ndt_cv.fit(X_train, y_train)\nprint(dt_cv.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:22:29.323847Z","iopub.execute_input":"2022-01-15T23:22:29.324107Z","iopub.status.idle":"2022-01-15T23:27:19.366875Z","shell.execute_reply.started":"2022-01-15T23:22:29.324077Z","shell.execute_reply":"2022-01-15T23:27:19.366071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n# fit the model\n# Decision Tree\nstart_time = time.time()\nkwargs_regularize = dict(criterion='gini',\n                         max_depth=5,\n                         max_leaf_nodes=14,\n                         min_samples_split=2)\ndt = tree.DecisionTreeClassifier(**kwargs_regularize)\ndt.fit(X_train, y_train)\n\ndt_time = time.time() - start_time\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:33:50.396320Z","iopub.execute_input":"2022-01-15T23:33:50.397113Z","iopub.status.idle":"2022-01-15T23:33:50.973768Z","shell.execute_reply.started":"2022-01-15T23:33:50.397070Z","shell.execute_reply":"2022-01-15T23:33:50.972171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Tree Visualization**","metadata":{}},{"cell_type":"code","source":"%pip install pydotplus","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:36:53.399744Z","iopub.execute_input":"2022-01-15T23:36:53.400224Z","iopub.status.idle":"2022-01-15T23:37:02.946196Z","shell.execute_reply.started":"2022-01-15T23:36:53.400186Z","shell.execute_reply":"2022-01-15T23:37:02.945404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for decision tree visualizaiton\nfrom six import StringIO\nfrom IPython.display import Image\nfrom sklearn.tree import export_graphviz\nimport pydotplus\n\ndot_data = StringIO()\nexport_graphviz(dt, out_file=dot_data,\n                filled=True, rounded=True,\n                feature_names=X_train.columns.values,\n                class_names=['human', 'bot'],\n                special_characters=True)\ngraph = pydotplus.graph_from_dot_data(dot_data.getvalue())\nImage(graph.create_png())","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:37:07.625562Z","iopub.execute_input":"2022-01-15T23:37:07.626380Z","iopub.status.idle":"2022-01-15T23:37:08.925808Z","shell.execute_reply.started":"2022-01-15T23:37:07.626321Z","shell.execute_reply":"2022-01-15T23:37:08.922660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check feature importance and display in bar plot.\nprint('Feature importance of Decision Tree Model')\nplt.style.use('ggplot')\nfig = plt.figure(figsize=(5, 5))\nfeat_importances = pd.Series(dt.feature_importances_, index=X_train.columns)\nfeat_importances.nsmallest(5).plot(kind='barh', alpha=0.7)\nfig.savefig('dt_feature.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:37:37.876239Z","iopub.execute_input":"2022-01-15T23:37:37.876767Z","iopub.status.idle":"2022-01-15T23:37:38.109550Z","shell.execute_reply.started":"2022-01-15T23:37:37.876727Z","shell.execute_reply":"2022-01-15T23:37:38.108887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Evaluation Metric**","metadata":{}},{"cell_type":"code","source":"# predict\ny_dt_pred = dt.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:38:15.051526Z","iopub.execute_input":"2022-01-15T23:38:15.052085Z","iopub.status.idle":"2022-01-15T23:38:15.091843Z","shell.execute_reply.started":"2022-01-15T23:38:15.052046Z","shell.execute_reply":"2022-01-15T23:38:15.091097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Accuracy**","metadata":{}},{"cell_type":"code","source":"# for evaluation metric\n# accuracy\nfrom sklearn.metrics import accuracy_score\n# accuracy score\nprint(f\"Decision Tree Accuracy: {accuracy_score(y_dt_pred, y_test):.3f}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:39:02.642165Z","iopub.execute_input":"2022-01-15T23:39:02.642450Z","iopub.status.idle":"2022-01-15T23:39:02.713308Z","shell.execute_reply.started":"2022-01-15T23:39:02.642419Z","shell.execute_reply":"2022-01-15T23:39:02.712565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**AUC**","metadata":{}},{"cell_type":"code","source":"# AUC\nfrom sklearn.metrics import roc_curve, auc\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\nfrom pylab import rcParams\n\n# Plot ROC in one graph\ny_dt_score = dt.predict_proba(X_test)[:, 1]\nfpr_dt, tpr_dt, _dt = roc_curve(y_test, y_dt_score)\nroc_dt_auc = auc(fpr_dt, tpr_dt)\n\nfig = plt.figure(figsize=(5, 5))\nplt.plot(fpr_dt, tpr_dt, label='DT ROC curve (area = %0.2f)' % roc_dt_auc)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.005])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic')\nplt.legend(loc=\"lower right\")\nplt.show()\nfig.savefig('roc_dt_auc.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:40:10.869355Z","iopub.execute_input":"2022-01-15T23:40:10.870109Z","iopub.status.idle":"2022-01-15T23:40:11.221675Z","shell.execute_reply.started":"2022-01-15T23:40:10.870072Z","shell.execute_reply":"2022-01-15T23:40:11.220935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Pecision vs Recall**","metadata":{}},{"cell_type":"code","source":"print('Classification Report of Decision Tree Model')\nprint(classification_report(y_test, y_dt_pred))","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:55:30.307188Z","iopub.execute_input":"2022-01-15T23:55:30.307485Z","iopub.status.idle":"2022-01-15T23:55:31.579938Z","shell.execute_reply.started":"2022-01-15T23:55:30.307452Z","shell.execute_reply":"2022-01-15T23:55:31.579061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Second Model: Random Forest**","metadata":{}},{"cell_type":"code","source":"# for random search\nfrom sklearn.ensemble import RandomForestClassifier\n\nrf = RandomForestClassifier(n_estimators=50)\nparam_grid = {\n    'max_depth': range(3, 6),\n    'max_leaf_nodes': range(8, 12),\n    'max_features': ['sqrt', 'auto', 'log2']\n}\n\nrf_cv = GridSearchCV(estimator=rf,\n                     param_grid=param_grid,\n                     cv=5)\nrf_cv.fit(X_train, y_train)\nprint(rf_cv.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T23:56:28.345223Z","iopub.execute_input":"2022-01-15T23:56:28.345775Z","iopub.status.idle":"2022-01-16T00:22:36.996889Z","shell.execute_reply.started":"2022-01-15T23:56:28.345737Z","shell.execute_reply":"2022-01-16T00:22:36.996073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\n\nrf = RandomForestClassifier(n_estimators=100, max_depth=5,\n                            max_leaf_nodes=11, max_features='log2',\n                            bootstrap=True, oob_score=True)\nrf.fit(X_train, y_train)\n\nrf_time = time.time() - start_time\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:22:36.998707Z","iopub.execute_input":"2022-01-16T00:22:36.998959Z","iopub.status.idle":"2022-01-16T00:23:03.315879Z","shell.execute_reply.started":"2022-01-16T00:22:36.998925Z","shell.execute_reply":"2022-01-16T00:23:03.314401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check feature importance and display in bar plot.\nprint('Feature importance of Random Forest Model')\nplt.style.use('ggplot')\nfig = plt.figure(figsize=(5, 5))\nfeat_importances = pd.Series(rf.feature_importances_, index=X_train.columns)\nfeat_importances.nsmallest(5).plot(kind='barh', alpha=0.7)\nfig.savefig('rf_feature.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:23:03.317275Z","iopub.execute_input":"2022-01-16T00:23:03.317553Z","iopub.status.idle":"2022-01-16T00:23:03.550901Z","shell.execute_reply.started":"2022-01-16T00:23:03.317517Z","shell.execute_reply":"2022-01-16T00:23:03.550286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Evaluation Metric**\n\nAccuracy","metadata":{}},{"cell_type":"code","source":"y_rf_pred = rf.predict(X_test)\nprint(f\"Random Forest Accuracy: {accuracy_score(y_rf_pred, y_test):.3f}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:23:03.552937Z","iopub.execute_input":"2022-01-16T00:23:03.553181Z","iopub.status.idle":"2022-01-16T00:23:06.930911Z","shell.execute_reply.started":"2022-01-16T00:23:03.553148Z","shell.execute_reply":"2022-01-16T00:23:06.930057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"AUC","metadata":{}},{"cell_type":"code","source":"# Plot ROC in one graph\ny_rf_score = rf.predict_proba(X_test)[:, 1]\nfpr_rf, tpr_rf, _rf = roc_curve(y_test, y_rf_score)\nroc_rf_auc = auc(fpr_rf, tpr_rf)\n\nplt.figure(figsize=(5, 5))\nplt.plot(fpr_rf, tpr_rf, label='RF ROC curve (area = %0.2f)' % roc_rf_auc)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.005])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic')\nplt.legend(loc=\"lower right\")\nplt.show()\nfig.savefig('roc_rf_auc.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:23:06.933413Z","iopub.execute_input":"2022-01-16T00:23:06.933633Z","iopub.status.idle":"2022-01-16T00:23:10.338316Z","shell.execute_reply.started":"2022-01-16T00:23:06.933599Z","shell.execute_reply":"2022-01-16T00:23:10.337673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Precision vs. Recall","metadata":{}},{"cell_type":"code","source":"print('Classification Report of Random Forest Model')\nprint(classification_report(y_test, y_rf_pred))","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:23:10.339412Z","iopub.execute_input":"2022-01-16T00:23:10.339694Z","iopub.status.idle":"2022-01-16T00:23:11.618046Z","shell.execute_reply.started":"2022-01-16T00:23:10.339658Z","shell.execute_reply":"2022-01-16T00:23:11.617108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Third Model: Gradient Boosting**","metadata":{}},{"cell_type":"code","source":"# for gradient boosting\nfrom sklearn.ensemble import GradientBoostingClassifier\n\ngb = GradientBoostingClassifier(n_estimators=10)\nparam_grid = {\n    'max_depth': range(3, 6),\n    'max_leaf_nodes': range(8, 11)\n}\n\ngb_cv = GridSearchCV(estimator=gb,\n                     param_grid=param_grid,\n                     cv=5)\ngb_cv.fit(X_train, y_train)\nprint(gb_cv.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:23:11.620482Z","iopub.execute_input":"2022-01-16T00:23:11.620971Z","iopub.status.idle":"2022-01-16T00:26:45.907272Z","shell.execute_reply.started":"2022-01-16T00:23:11.620934Z","shell.execute_reply":"2022-01-16T00:26:45.906532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\n\ngb = GradientBoostingClassifier(n_estimators=100, max_depth=5, max_features='sqrt',\n                                max_leaf_nodes=9)\ngb.fit(X_train, y_train)\n\ngb_time = time.time() - start_time\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:26:45.908636Z","iopub.execute_input":"2022-01-16T00:26:45.909038Z","iopub.status.idle":"2022-01-16T00:27:21.002502Z","shell.execute_reply.started":"2022-01-16T00:26:45.909001Z","shell.execute_reply":"2022-01-16T00:27:21.001626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check feature importance and display in bar plot.\nprint('Feature importance of Gradient Boosting Model')\nplt.style.use('ggplot')\nfig = plt.figure(figsize=(5, 5))\nfeat_importances = pd.Series(gb.feature_importances_, index=X_train.columns)\nfeat_importances.nsmallest(5).plot(kind='barh', alpha=0.7)\nfig.savefig('gb_feature.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:27:21.007108Z","iopub.execute_input":"2022-01-16T00:27:21.009881Z","iopub.status.idle":"2022-01-16T00:27:21.380185Z","shell.execute_reply.started":"2022-01-16T00:27:21.009838Z","shell.execute_reply":"2022-01-16T00:27:21.379418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Evaluation Metrics**\n\nAccuracy","metadata":{}},{"cell_type":"code","source":"y_gb_pred = gb.predict(X_test)\nprint(f\"Gradient Boosting Accuracy: {accuracy_score(y_gb_pred, y_test):.3f}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:27:21.384612Z","iopub.execute_input":"2022-01-16T00:27:21.384986Z","iopub.status.idle":"2022-01-16T00:27:22.939170Z","shell.execute_reply.started":"2022-01-16T00:27:21.384952Z","shell.execute_reply":"2022-01-16T00:27:22.937621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"AUC","metadata":{}},{"cell_type":"code","source":"# Plot ROC in one graph\ny_gb_score = gb.predict_proba(X_test)[:, 1]\nfpr_gb, tpr_gb, _gb = roc_curve(y_test, y_gb_score)\nroc_gb_auc = auc(fpr_gb, tpr_gb)\n\nplt.figure(figsize=(5, 5))\nplt.plot(fpr_gb, tpr_gb, label='GB ROC curve (area = %0.2f)' % roc_gb_auc)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.005])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic')\nplt.legend(loc=\"lower right\")\nplt.show()\nfig.savefig('roc_gb_auc.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:27:22.940453Z","iopub.execute_input":"2022-01-16T00:27:22.940708Z","iopub.status.idle":"2022-01-16T00:27:24.224662Z","shell.execute_reply.started":"2022-01-16T00:27:22.940673Z","shell.execute_reply":"2022-01-16T00:27:24.224012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Precison vs. Recall","metadata":{}},{"cell_type":"code","source":"print('Classification Report of Gradient Boosting Model')\nprint(classification_report(y_test, y_gb_pred))","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:27:24.225996Z","iopub.execute_input":"2022-01-16T00:27:24.226225Z","iopub.status.idle":"2022-01-16T00:27:25.455211Z","shell.execute_reply.started":"2022-01-16T00:27:24.226192Z","shell.execute_reply":"2022-01-16T00:27:25.454443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**8 Compare Different Models**","metadata":{}},{"cell_type":"code","source":"# for model comparision\nfrom sklearn import metrics\n\ndt_metrics = [metrics.accuracy_score(y_test, y_dt_pred), metrics.precision_score(y_test, y_dt_pred),\n              metrics.recall_score(y_test, y_dt_pred), metrics.f1_score(\n                  y_test, y_dt_pred),\n              metrics.roc_auc_score(y_test, y_dt_pred), dt_time]\nrf_metrics = [metrics.accuracy_score(y_test, y_rf_pred), metrics.precision_score(y_test, y_rf_pred),\n              metrics.recall_score(y_test, y_rf_pred), metrics.f1_score(\n                  y_test, y_rf_pred),\n              metrics.roc_auc_score(y_test, y_rf_pred), rf_time]\ngb_metrics = [metrics.accuracy_score(y_test, y_gb_pred), metrics.precision_score(y_test, y_gb_pred),\n              metrics.recall_score(y_test, y_gb_pred), metrics.f1_score(\n                  y_test, y_gb_pred),\n              metrics.roc_auc_score(y_test, y_gb_pred), gb_time]","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:40:11.591484Z","iopub.execute_input":"2022-01-16T00:40:11.592388Z","iopub.status.idle":"2022-01-16T00:40:14.675688Z","shell.execute_reply.started":"2022-01-16T00:40:11.592328Z","shell.execute_reply":"2022-01-16T00:40:14.674783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 8))\nindex = np.arange(5)\nwidth = 0.2\nb1 = plt.bar(index, dt_metrics[0:5], width,\n             alpha=0.4, color='grey', label='decision tree')\nb2 = plt.bar(index+width, rf_metrics[0:5], width,\n             alpha=0.8, color='powderblue', label='random forest')\nb3 = plt.bar(index+2*width, gb_metrics[0:5], width,\n             alpha=0.8, color='pink', label='gradient boosting')\nplt.title('Model Comparison')\nplt.ylabel('score')\nplt.xticks(index+width, ('accuracy', 'precision', 'recall', 'F1', 'ROC AUC'))\nplt.legend(loc=8, ncol=3, mode=\"expand\", borderaxespad=0.)\nplt.show()\nfig.savefig('model_comparison.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:40:18.501738Z","iopub.execute_input":"2022-01-16T00:40:18.502488Z","iopub.status.idle":"2022-01-16T00:40:18.775238Z","shell.execute_reply.started":"2022-01-16T00:40:18.502448Z","shell.execute_reply":"2022-01-16T00:40:18.774593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = [\"Accuracy_score\", \"Precision_score\",\n         \"Recall_Score\", \"F1_score\", \"ROC_AUC_score\", \"Time(s)\"]\ntable = pd.DataFrame({'Decision Tree': dt_metrics,\n                      'Random Forest': rf_metrics, 'Gradient Boosting': gb_metrics})\ntable = table.transpose()\ntable.columns = label\ntable.transpose().round(3)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T00:40:23.291820Z","iopub.execute_input":"2022-01-16T00:40:23.292098Z","iopub.status.idle":"2022-01-16T00:40:23.308270Z","shell.execute_reply.started":"2022-01-16T00:40:23.292053Z","shell.execute_reply":"2022-01-16T00:40:23.307353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion**\n\nFrom the feature importance, we can see that in the online bidding, human being are more likely come from a random url into a certain aution, however, the robots are more likely come from few specific url. And compared with human beings robots are more likely to use different ip addresses for online bidding. Thus if for a user, it using multi-ip addresses and come from few certain url, we might consider it is a robots.","metadata":{}}]}