{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":4294,"databundleVersionId":33829}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Importing Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport datetime\n\n\npd.set_option('display.max_columns', None)\npd.set_option('display.float_format', lambda x: '%.3f' % x)\n\nimport matplotlib.pyplot as mp\nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:39:53.309122Z","iopub.execute_input":"2024-02-26T17:39:53.309989Z","iopub.status.idle":"2024-02-26T17:39:54.620053Z","shell.execute_reply.started":"2024-02-26T17:39:53.309933Z","shell.execute_reply":"2024-02-26T17:39:54.618901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data = pd.read_csv(\"/kaggle/input/facebook-recruiting-iv-human-or-bot/bids.csv.zip\")\nbids_data","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:39:56.584746Z","iopub.execute_input":"2024-02-26T17:39:56.585958Z","iopub.status.idle":"2024-02-26T17:40:26.522226Z","shell.execute_reply.started":"2024-02-26T17:39:56.585905Z","shell.execute_reply":"2024-02-26T17:40:26.520665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_data = pd.read_csv(\"/kaggle/input/facebook-recruiting-iv-human-or-bot/train.csv.zip\")\nbidder_train_data","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:40:37.512953Z","iopub.execute_input":"2024-02-26T17:40:37.513434Z","iopub.status.idle":"2024-02-26T17:40:37.548497Z","shell.execute_reply.started":"2024-02-26T17:40:37.513399Z","shell.execute_reply":"2024-02-26T17:40:37.547127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_test_data = pd.read_csv(\"/kaggle/input/facebook-recruiting-iv-human-or-bot/test.csv.zip\")\nbidder_test_data","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:40:41.417028Z","iopub.execute_input":"2024-02-26T17:40:41.417466Z","iopub.status.idle":"2024-02-26T17:40:41.461583Z","shell.execute_reply.started":"2024-02-26T17:40:41.417432Z","shell.execute_reply":"2024-02-26T17:40:41.460300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Cleaning","metadata":{}},{"cell_type":"markdown","source":"### Null Values","metadata":{}},{"cell_type":"code","source":"bids_data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:40:45.280835Z","iopub.execute_input":"2024-02-26T17:40:45.281303Z","iopub.status.idle":"2024-02-26T17:40:51.063506Z","shell.execute_reply.started":"2024-02-26T17:40:45.281266Z","shell.execute_reply":"2024-02-26T17:40:51.062054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data['country'].mode()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:40:51.065991Z","iopub.execute_input":"2024-02-26T17:40:51.066443Z","iopub.status.idle":"2024-02-26T17:40:52.533125Z","shell.execute_reply.started":"2024-02-26T17:40:51.066400Z","shell.execute_reply":"2024-02-26T17:40:52.531658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data['country'].fillna(bids_data['country'].mode()[0], inplace=True)\nbids_data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:40:52.535804Z","iopub.execute_input":"2024-02-26T17:40:52.536270Z","iopub.status.idle":"2024-02-26T17:41:00.680613Z","shell.execute_reply.started":"2024-02-26T17:40:52.536231Z","shell.execute_reply":"2024-02-26T17:41:00.679415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_data.isnull().sum()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-02-26T17:41:00.682765Z","iopub.execute_input":"2024-02-26T17:41:00.683112Z","iopub.status.idle":"2024-02-26T17:41:00.692921Z","shell.execute_reply.started":"2024-02-26T17:41:00.683081Z","shell.execute_reply":"2024-02-26T17:41:00.691820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_test_data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:00.694314Z","iopub.execute_input":"2024-02-26T17:41:00.695320Z","iopub.status.idle":"2024-02-26T17:41:00.709199Z","shell.execute_reply.started":"2024-02-26T17:41:00.695277Z","shell.execute_reply":"2024-02-26T17:41:00.707801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculation of unique values for bidder ids\n\nCalculating the unique counts of the bidder information and merging it in the main data frame.","metadata":{}},{"cell_type":"code","source":"bidder_unique_ids = bids_data.groupby('bidder_id').nunique().reset_index()\nbidder_unique_ids","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:12.370670Z","iopub.execute_input":"2024-02-26T17:41:12.371051Z","iopub.status.idle":"2024-02-26T17:41:30.696562Z","shell.execute_reply.started":"2024-02-26T17:41:12.371022Z","shell.execute_reply":"2024-02-26T17:41:30.695413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids = bidder_train_data.merge(bidder_unique_ids, how='left')\nbidder_train_unique_ids.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:30.698832Z","iopub.execute_input":"2024-02-26T17:41:30.700707Z","iopub.status.idle":"2024-02-26T17:41:30.732526Z","shell.execute_reply.started":"2024-02-26T17:41:30.700663Z","shell.execute_reply":"2024-02-26T17:41:30.731356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids.rename(columns = {'bid_id': 'bids_count', 'auction': 'auction_count', 'merchandise': 'merch_count', 'device': 'device_count', 'country': 'country_count', 'ip': 'ip_count', 'url': 'url_count'}, inplace=True)\nbidder_train_unique_ids.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:30.734370Z","iopub.execute_input":"2024-02-26T17:41:30.735177Z","iopub.status.idle":"2024-02-26T17:41:30.757861Z","shell.execute_reply.started":"2024-02-26T17:41:30.735135Z","shell.execute_reply":"2024-02-26T17:41:30.756475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_test_unique_ids = bidder_test_data.merge(bidder_unique_ids, how='left')\nbidder_test_unique_ids.rename(columns = {'bid_id': 'bids_count', 'auction': 'auction_count', 'merchandise': 'merch_count', 'device': 'device_count', 'country': 'country_count', 'ip': 'ip_count', 'url': 'url_count'}, inplace=True)\nbidder_test_unique_ids.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:30.760915Z","iopub.execute_input":"2024-02-26T17:41:30.761519Z","iopub.status.idle":"2024-02-26T17:41:30.793624Z","shell.execute_reply.started":"2024-02-26T17:41:30.761477Z","shell.execute_reply":"2024-02-26T17:41:30.792508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculating the differences between consecutive bids.\n\nCalculating the time differences between consecutive bids and using that to find out which bids were made simultaneously and that can be used to indicate a bot behaviour.","metadata":{}},{"cell_type":"code","source":"bids_data_time_sorted = bids_data.sort_values(['bidder_id', 'time'])\nbids_data_time_sorted.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:35.381769Z","iopub.execute_input":"2024-02-26T17:41:35.382505Z","iopub.status.idle":"2024-02-26T17:41:45.847661Z","shell.execute_reply.started":"2024-02-26T17:41:35.382466Z","shell.execute_reply":"2024-02-26T17:41:45.846416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data_time_diff = bids_data_time_sorted.groupby('bidder_id')[['time']].diff()\nbids_data_time_diff.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:45.849860Z","iopub.execute_input":"2024-02-26T17:41:45.850199Z","iopub.status.idle":"2024-02-26T17:41:47.336111Z","shell.execute_reply.started":"2024-02-26T17:41:45.850171Z","shell.execute_reply":"2024-02-26T17:41:47.334888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data_time_diff['time'] = bids_data_time_diff['time'].fillna(0)\nbids_data_time_diff.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:47.337778Z","iopub.execute_input":"2024-02-26T17:41:47.338214Z","iopub.status.idle":"2024-02-26T17:41:47.410664Z","shell.execute_reply.started":"2024-02-26T17:41:47.338164Z","shell.execute_reply":"2024-02-26T17:41:47.409535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data_time_sorted['time_diff'] = bids_data_time_diff[['time']]\nbids_data_time_sorted.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:41:57.751902Z","iopub.execute_input":"2024-02-26T17:41:57.752343Z","iopub.status.idle":"2024-02-26T17:41:57.835479Z","shell.execute_reply.started":"2024-02-26T17:41:57.752293Z","shell.execute_reply":"2024-02-26T17:41:57.834133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data_time_sorted_intervals = bids_data_time_sorted.groupby('bidder_id')[['time_diff']].describe().reset_index()\nbids_data_time_sorted_intervals.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:42:00.682669Z","iopub.execute_input":"2024-02-26T17:42:00.683081Z","iopub.status.idle":"2024-02-26T17:42:18.577493Z","shell.execute_reply.started":"2024-02-26T17:42:00.683043Z","shell.execute_reply":"2024-02-26T17:42:18.576429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data_time_sorted_intervals = bids_data_time_sorted_intervals.droplevel(axis=1, level=0)\nbids_data_time_sorted_intervals.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:42:18.579838Z","iopub.execute_input":"2024-02-26T17:42:18.581040Z","iopub.status.idle":"2024-02-26T17:42:18.603880Z","shell.execute_reply.started":"2024-02-26T17:42:18.580994Z","shell.execute_reply":"2024-02-26T17:42:18.602681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data_time_sorted_intervals = bids_data_time_sorted_intervals.fillna(0)\nbids_data_time_sorted_intervals['iqr'] = bids_data_time_sorted_intervals['75%'] - bids_data_time_sorted_intervals['25%']\n# bids_data_time_sorted_intervals = bids_data_time_sorted_intervals.drop(['25%', '75%', 'count'], axis = 1)\nbids_data_time_sorted_intervals.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:42:18.605900Z","iopub.execute_input":"2024-02-26T17:42:18.606690Z","iopub.status.idle":"2024-02-26T17:42:18.633723Z","shell.execute_reply.started":"2024-02-26T17:42:18.606642Z","shell.execute_reply":"2024-02-26T17:42:18.632347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data_time_sorted_intervals.rename(columns={\"\":\"bidder_id\"}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:42:18.636111Z","iopub.execute_input":"2024-02-26T17:42:18.636540Z","iopub.status.idle":"2024-02-26T17:42:18.642575Z","shell.execute_reply.started":"2024-02-26T17:42:18.636507Z","shell.execute_reply":"2024-02-26T17:42:18.641380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:42:45.639925Z","iopub.execute_input":"2024-02-26T17:42:45.640356Z","iopub.status.idle":"2024-02-26T17:42:45.659038Z","shell.execute_reply.started":"2024-02-26T17:42:45.640310Z","shell.execute_reply":"2024-02-26T17:42:45.657787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bids_data_time_sorted_intervals.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:42:49.831096Z","iopub.execute_input":"2024-02-26T17:42:49.831553Z","iopub.status.idle":"2024-02-26T17:42:49.851089Z","shell.execute_reply.started":"2024-02-26T17:42:49.831510Z","shell.execute_reply":"2024-02-26T17:42:49.849674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:42:52.431768Z","iopub.execute_input":"2024-02-26T17:42:52.432205Z","iopub.status.idle":"2024-02-26T17:42:52.449355Z","shell.execute_reply.started":"2024-02-26T17:42:52.432172Z","shell.execute_reply":"2024-02-26T17:42:52.447874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids = bidder_train_unique_ids.merge(bids_data_time_sorted_intervals, on='bidder_id', how='left')\nbidder_train_unique_ids = bidder_train_unique_ids.fillna(bidder_train_unique_ids.median(numeric_only=True))\nbidder_train_unique_ids.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:43:05.581168Z","iopub.execute_input":"2024-02-26T17:43:05.582508Z","iopub.status.idle":"2024-02-26T17:43:05.630067Z","shell.execute_reply.started":"2024-02-26T17:43:05.582455Z","shell.execute_reply":"2024-02-26T17:43:05.628576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_test_unique_ids = bidder_test_unique_ids.merge(bids_data_time_sorted_intervals, on='bidder_id', how='left')\nbidder_test_unique_ids = bidder_test_unique_ids.fillna(bidder_test_unique_ids.median(numeric_only=True))\nbidder_test_unique_ids.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:53:37.041858Z","iopub.execute_input":"2024-02-26T17:53:37.042344Z","iopub.status.idle":"2024-02-26T17:53:37.091995Z","shell.execute_reply.started":"2024-02-26T17:53:37.042295Z","shell.execute_reply":"2024-02-26T17:53:37.090307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Bids in last half since bots would try to outbid other members who started bidding in the beginning.","metadata":{}},{"cell_type":"code","source":"sorted_by_auction_df = bids_data.sort_values(['auction', 'time'])[['bidder_id', 'auction', 'time']]\nsorted_by_auction_df_aggregated = sorted_by_auction_df[['auction', 'time']].groupby('auction').agg([max, min])\nsorted_by_auction_df_aggregated.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:53:46.973871Z","iopub.execute_input":"2024-02-26T17:53:46.974319Z","iopub.status.idle":"2024-02-26T17:54:01.272643Z","shell.execute_reply.started":"2024-02-26T17:53:46.974276Z","shell.execute_reply":"2024-02-26T17:54:01.271364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_by_auction_df_aggregated = sorted_by_auction_df_aggregated.reset_index().droplevel(axis=1, level=0)\nsorted_by_auction_df_aggregated.rename(columns = {'':'auction'}, inplace=True)\nsorted_by_auction_df_aggregated.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:54:04.428433Z","iopub.execute_input":"2024-02-26T17:54:04.428959Z","iopub.status.idle":"2024-02-26T17:54:04.450237Z","shell.execute_reply.started":"2024-02-26T17:54:04.428825Z","shell.execute_reply":"2024-02-26T17:54:04.448586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_by_auction_df_aggregated['total_auction_time'] = sorted_by_auction_df_aggregated['max'] - sorted_by_auction_df_aggregated['min']\nsorted_by_auction_df_aggregated.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:54:07.045170Z","iopub.execute_input":"2024-02-26T17:54:07.045646Z","iopub.status.idle":"2024-02-26T17:54:07.061899Z","shell.execute_reply.started":"2024-02-26T17:54:07.045610Z","shell.execute_reply":"2024-02-26T17:54:07.060508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_by_auction_df_final = sorted_by_auction_df.merge(sorted_by_auction_df_aggregated, on='auction', how='left')\nsorted_by_auction_df_final.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:54:08.963413Z","iopub.execute_input":"2024-02-26T17:54:08.963917Z","iopub.status.idle":"2024-02-26T17:54:14.260308Z","shell.execute_reply.started":"2024-02-26T17:54:08.963879Z","shell.execute_reply":"2024-02-26T17:54:14.258787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_by_auction_df_final['second_half'] = (sorted_by_auction_df_final['time'] - sorted_by_auction_df_final['min']) > (sorted_by_auction_df_final['max'] - sorted_by_auction_df_final['time'])\nsorted_by_auction_df_final['first_half'] = (sorted_by_auction_df_final['time'] - sorted_by_auction_df_final['min']) < (sorted_by_auction_df_final['max'] - sorted_by_auction_df_final['time'])\nsorted_by_auction_df_final.head(10)\n# sorted_by_auction_df_final[(sorted_by_auction_df_final['time'] - (sorted_by_auction_df_final['min'] + sorted_by_auction_df_final['total_auction_time']/2)) < 0]","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:54:22.356317Z","iopub.execute_input":"2024-02-26T17:54:22.356820Z","iopub.status.idle":"2024-02-26T17:54:22.483122Z","shell.execute_reply.started":"2024-02-26T17:54:22.356781Z","shell.execute_reply":"2024-02-26T17:54:22.481556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auction_stats = sorted_by_auction_df_final[['bidder_id', 'second_half']].groupby('bidder_id').agg(['count', sum]).reset_index().droplevel(axis=1, level=0)\nauction_stats.rename(columns = {'': 'bidder_id', 'count': 'total_bids', 'sum': 'total_second_half_bids'}, inplace=True)\nauction_stats['total_first_half_bids'] = auction_stats['total_bids'] - auction_stats['total_second_half_bids']\nauction_stats['percent_second_half_bids'] = (auction_stats['total_second_half_bids'] / auction_stats['total_bids'])*100\nauction_stats['percent_first_half_bids'] = (auction_stats['total_first_half_bids'] / auction_stats['total_bids'])*100\nauction_stats.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:54:31.229508Z","iopub.execute_input":"2024-02-26T17:54:31.229930Z","iopub.status.idle":"2024-02-26T17:54:33.400265Z","shell.execute_reply.started":"2024-02-26T17:54:31.229899Z","shell.execute_reply":"2024-02-26T17:54:33.398708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids = bidder_train_unique_ids.merge(auction_stats, on='bidder_id', how='left')\nbidder_train_unique_ids = bidder_train_unique_ids.fillna(bidder_train_unique_ids.median(numeric_only=True))\nbidder_train_unique_ids.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:54:54.272818Z","iopub.execute_input":"2024-02-26T17:54:54.273260Z","iopub.status.idle":"2024-02-26T17:54:54.320995Z","shell.execute_reply.started":"2024-02-26T17:54:54.273226Z","shell.execute_reply":"2024-02-26T17:54:54.320108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_test_unique_ids = bidder_test_unique_ids.merge(auction_stats, on='bidder_id', how='left')\nbidder_test_unique_ids = bidder_test_unique_ids.fillna(bidder_test_unique_ids.median(numeric_only=True))\nbidder_test_unique_ids.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:55:50.979143Z","iopub.execute_input":"2024-02-26T17:55:50.979613Z","iopub.status.idle":"2024-02-26T17:55:51.018886Z","shell.execute_reply.started":"2024-02-26T17:55:50.979578Z","shell.execute_reply":"2024-02-26T17:55:51.017440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number of times made HIGHEST bids in auction","metadata":{}},{"cell_type":"code","source":"total_bids_in_auction = bids_data.groupby(['bidder_id', 'auction']).count().reset_index()\nnumber_of_highest_bids_in_aucts = total_bids_in_auction.groupby(['auction']).max()[['bidder_id', 'bid_id']].reset_index()\nnumber_of_highest_bids_in_aucts = number_of_highest_bids_in_aucts.groupby('bidder_id').count()[['bid_id']].reset_index().rename(columns={'bid_id':'num_aucts_with_highest_bids'}, errors='raise')\n\nnumber_of_highest_bids_in_aucts.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:56:08.508613Z","iopub.execute_input":"2024-02-26T17:56:08.509067Z","iopub.status.idle":"2024-02-26T17:56:16.969084Z","shell.execute_reply.started":"2024-02-26T17:56:08.509035Z","shell.execute_reply":"2024-02-26T17:56:16.967536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids = bidder_train_unique_ids.merge(number_of_highest_bids_in_aucts, on='bidder_id', how='left')\nbidder_train_unique_ids['num_aucts_with_highest_bids'] = bidder_train_unique_ids['num_aucts_with_highest_bids'].fillna(0)\n# bidder_train_unique_ids.head()\n\n\n\nbidder_test_unique_ids = bidder_test_unique_ids.merge(number_of_highest_bids_in_aucts, on='bidder_id', how='left')\nbidder_test_unique_ids['num_aucts_with_highest_bids'] = bidder_test_unique_ids['num_aucts_with_highest_bids'].fillna(0)\n# bidder_test_unique_ids.head(10)\n\n# train_set = train_set.merge(max_bids_in_auct, on='bidder_id', how='left').fillna(0)\n\n# test_set = test_set.merge(max_bids_in_auct, on='bidder_id', how='left').fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:56:39.273972Z","iopub.execute_input":"2024-02-26T17:56:39.274452Z","iopub.status.idle":"2024-02-26T17:56:39.298712Z","shell.execute_reply.started":"2024-02-26T17:56:39.274415Z","shell.execute_reply":"2024-02-26T17:56:39.297372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Max num of bids across all auctions.","metadata":{}},{"cell_type":"code","source":"total_bids_in_auction = bids_data.groupby(['bidder_id', 'auction']).count().reset_index()[['bidder_id', 'auction', 'bid_id']].rename(columns = {'bid_id': 'max_bids_in_auct'})\ntotal_bids_in_auction = total_bids_in_auction[['bidder_id', 'max_bids_in_auct']].groupby('bidder_id').max().reset_index()\ntotal_bids_in_auction.head()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-02-26T17:58:57.574984Z","iopub.execute_input":"2024-02-26T17:58:57.575465Z","iopub.status.idle":"2024-02-26T17:59:04.925419Z","shell.execute_reply.started":"2024-02-26T17:58:57.575430Z","shell.execute_reply":"2024-02-26T17:59:04.923923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids = bidder_train_unique_ids.merge(total_bids_in_auction, on='bidder_id', how='left')\nbidder_train_unique_ids['max_bids_in_auct'] = bidder_train_unique_ids['max_bids_in_auct'].fillna(0)\n\nbidder_test_unique_ids = bidder_test_unique_ids.merge(total_bids_in_auction, on='bidder_id', how='left')\nbidder_test_unique_ids['max_bids_in_auct'] = bidder_test_unique_ids['max_bids_in_auct'].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:59:04.927579Z","iopub.execute_input":"2024-02-26T17:59:04.927923Z","iopub.status.idle":"2024-02-26T17:59:04.954213Z","shell.execute_reply.started":"2024-02-26T17:59:04.927894Z","shell.execute_reply":"2024-02-26T17:59:04.952940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"        ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of times first bids","metadata":{}},{"cell_type":"code","source":"bids_data_time_sorted = bids_data.sort_values(['time']).groupby(\"auction\").first().reset_index()\nbids_data_time_sorted.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:59:18.023314Z","iopub.execute_input":"2024-02-26T17:59:18.024639Z","iopub.status.idle":"2024-02-26T17:59:29.148795Z","shell.execute_reply.started":"2024-02-26T17:59:18.024598Z","shell.execute_reply":"2024-02-26T17:59:29.147044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_of_first_bids_in_aucts = bids_data_time_sorted.groupby(\"bidder_id\").count().reset_index()[[\"bidder_id\", 'bid_id']].rename(columns={'bid_id':'num_aucts_with_first_bids'}, errors='raise')\nnumber_of_first_bids_in_aucts.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:59:29.151092Z","iopub.execute_input":"2024-02-26T17:59:29.151615Z","iopub.status.idle":"2024-02-26T17:59:29.195194Z","shell.execute_reply.started":"2024-02-26T17:59:29.151571Z","shell.execute_reply":"2024-02-26T17:59:29.193772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids = bidder_train_unique_ids.merge(number_of_first_bids_in_aucts, on='bidder_id', how='left')\nbidder_train_unique_ids['num_aucts_with_first_bids'] = bidder_train_unique_ids['num_aucts_with_first_bids'].fillna(0)\n\nbidder_test_unique_ids = bidder_test_unique_ids.merge(number_of_first_bids_in_aucts, on='bidder_id', how='left')\nbidder_test_unique_ids['num_aucts_with_first_bids'] = bidder_test_unique_ids['num_aucts_with_first_bids'].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:59:32.817219Z","iopub.execute_input":"2024-02-26T17:59:32.817641Z","iopub.status.idle":"2024-02-26T17:59:32.840375Z","shell.execute_reply.started":"2024-02-26T17:59:32.817610Z","shell.execute_reply":"2024-02-26T17:59:32.838932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bids across devices","metadata":{}},{"cell_type":"code","source":"device_bids_auction_info = bids_data.groupby(['bidder_id', 'device']).nunique()[['bid_id', 'auction']].reset_index()\ndevice_bids_auction_info = device_bids_auction_info.rename(columns = {'bid_id': 'max_bids_on_any_device', 'auction': 'num_auct_on_any_device'})\ndevice_bids_auction_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T17:59:42.377843Z","iopub.execute_input":"2024-02-26T17:59:42.378309Z","iopub.status.idle":"2024-02-26T18:00:03.798022Z","shell.execute_reply.started":"2024-02-26T17:59:42.378275Z","shell.execute_reply":"2024-02-26T18:00:03.796793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_bids_auction_info = device_bids_auction_info.groupby('bidder_id').max().reset_index()\ndevice_bids_auction_info['max_bids_on_any_device_on_any_auct'] = device_bids_auction_info['max_bids_on_any_device'] / device_bids_auction_info['num_auct_on_any_device']\ndevice_bids_auction_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:00:07.684718Z","iopub.execute_input":"2024-02-26T18:00:07.685134Z","iopub.status.idle":"2024-02-26T18:00:08.420783Z","shell.execute_reply.started":"2024-02-26T18:00:07.685104Z","shell.execute_reply":"2024-02-26T18:00:08.419546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_bids_auction_info = device_bids_auction_info[['bidder_id', 'max_bids_on_any_device', 'max_bids_on_any_device_on_any_auct']]\ndevice_bids_auction_info.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:00:13.409979Z","iopub.execute_input":"2024-02-26T18:00:13.410384Z","iopub.status.idle":"2024-02-26T18:00:13.425372Z","shell.execute_reply.started":"2024-02-26T18:00:13.410354Z","shell.execute_reply":"2024-02-26T18:00:13.423641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids = bidder_train_unique_ids.merge(device_bids_auction_info, on='bidder_id', how='left')\nbidder_train_unique_ids[['max_bids_on_any_device_on_any_auct', \"max_bids_on_any_device\"]] = bidder_train_unique_ids[['max_bids_on_any_device_on_any_auct', \"max_bids_on_any_device\"]].fillna(0)\n\nbidder_test_unique_ids = bidder_test_unique_ids.merge(device_bids_auction_info, on='bidder_id', how='left')\nbidder_test_unique_ids[['max_bids_on_any_device_on_any_auct', \"max_bids_on_any_device\"]] = bidder_test_unique_ids[['max_bids_on_any_device_on_any_auct', \"max_bids_on_any_device\"]].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:00:17.756662Z","iopub.execute_input":"2024-02-26T18:00:17.757132Z","iopub.status.idle":"2024-02-26T18:00:17.789969Z","shell.execute_reply.started":"2024-02-26T18:00:17.757093Z","shell.execute_reply":"2024-02-26T18:00:17.788776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"bidder_train_unique_ids.columns","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:02:21.397375Z","iopub.execute_input":"2024-02-26T18:02:21.397991Z","iopub.status.idle":"2024-02-26T18:02:21.407783Z","shell.execute_reply.started":"2024-02-26T18:02:21.397946Z","shell.execute_reply":"2024-02-26T18:02:21.406613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'bids_count', 'auction_count', 'merch_count', 'device_count', 'time', 'country_count', 'ip_count', 'url_count'","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:02:23.155500Z","iopub.execute_input":"2024-02-26T18:02:23.155908Z","iopub.status.idle":"2024-02-26T18:02:23.164353Z","shell.execute_reply.started":"2024-02-26T18:02:23.155878Z","shell.execute_reply":"2024-02-26T18:02:23.162593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bidder_train_unique_ids['percent_concurrent_bids'] = bidder_train_unique_ids['num_concurrent_bids'] / bidder_train_unique_ids['bids_count']\nbidder_train_unique_ids['bids_per_auction'] = bidder_train_unique_ids['bids_count'] / bidder_train_unique_ids['auction_count']\nbidder_train_unique_ids['bids_per_device'] = bidder_train_unique_ids['bids_count'] / bidder_train_unique_ids['device_count']\nbidder_train_unique_ids['bids_per_url'] = bidder_train_unique_ids['bids_count'] / bidder_train_unique_ids['url_count']\nbidder_train_unique_ids['devices_per_auct'] = bidder_train_unique_ids['device_count'] / bidder_train_unique_ids['auction_count']\nbidder_train_unique_ids['ip_per_country'] = bidder_train_unique_ids['ip_count'] / bidder_train_unique_ids['country_count']\nbidder_train_unique_ids['percent_max_bids'] = bidder_train_unique_ids['max_bids_on_any_device'] / bidder_train_unique_ids['bids_count']\n\nbidder_train_unique_ids = bidder_train_unique_ids.fillna(0)\n\n# bidder_test_unique_ids['percent_concurrent_bids'] = bidder_test_unique_ids['num_concurrent_bids'] / bidder_test_unique_ids['bids_count']\nbidder_test_unique_ids['bids_per_auction'] = bidder_test_unique_ids['bids_count'] / bidder_test_unique_ids['auction_count']\nbidder_test_unique_ids['bids_per_device'] = bidder_test_unique_ids['bids_count'] / bidder_test_unique_ids['device_count']\nbidder_test_unique_ids['bids_per_url'] = bidder_test_unique_ids['bids_count'] / bidder_test_unique_ids['url_count']\nbidder_test_unique_ids['devices_per_auct'] = bidder_test_unique_ids['device_count'] / bidder_test_unique_ids['auction_count']\nbidder_test_unique_ids['ip_per_country'] = bidder_test_unique_ids['ip_count'] / bidder_test_unique_ids['country_count']\nbidder_test_unique_ids['percent_max_bids'] = bidder_test_unique_ids['max_bids_on_any_device'] / bidder_test_unique_ids['bids_count']\n\nbidder_test_unique_ids = bidder_test_unique_ids.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:02:44.312261Z","iopub.execute_input":"2024-02-26T18:02:44.312716Z","iopub.status.idle":"2024-02-26T18:02:44.338988Z","shell.execute_reply.started":"2024-02-26T18:02:44.312677Z","shell.execute_reply":"2024-02-26T18:02:44.337852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:02:54.388493Z","iopub.execute_input":"2024-02-26T18:02:54.388949Z","iopub.status.idle":"2024-02-26T18:02:54.395159Z","shell.execute_reply.started":"2024-02-26T18:02:54.388917Z","shell.execute_reply":"2024-02-26T18:02:54.393795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as mp\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:02:57.979532Z","iopub.execute_input":"2024-02-26T18:02:57.979947Z","iopub.status.idle":"2024-02-26T18:02:57.986027Z","shell.execute_reply.started":"2024-02-26T18:02:57.979914Z","shell.execute_reply":"2024-02-26T18:02:57.984399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids[bidder_train_unique_ids[\"outcome\"] == 1].sort_values(by=\"bids_count\")[[\"bidder_id\", \"bids_count\"]].head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:02:59.506477Z","iopub.execute_input":"2024-02-26T18:02:59.506932Z","iopub.status.idle":"2024-02-26T18:02:59.524925Z","shell.execute_reply.started":"2024-02-26T18:02:59.506899Z","shell.execute_reply":"2024-02-26T18:02:59.523721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mp.figure(figsize=(15,5))\nbot_bids_count = bidder_train_unique_ids[bidder_train_unique_ids[\"outcome\"] == 1][[\"bidder_id\", \"bids_count\"]].groupby(\"bids_count\").count().reset_index()\nax = sns.boxplot(x=bot_bids_count[\"bids_count\"])\n_ = ax.set_xticks(range(0, 180000, 5000))\n_ = mp.xticks(rotation = 90)\n# bot_bids_count","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:03:01.032400Z","iopub.execute_input":"2024-02-26T18:03:01.032857Z","iopub.status.idle":"2024-02-26T18:03:01.468677Z","shell.execute_reply.started":"2024-02-26T18:03:01.032823Z","shell.execute_reply":"2024-02-26T18:03:01.467389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Q.1) Should the bot representing the last data point be removed??\n\nQ.2) Should the bots with 1 bids be removed??\n\n**Currently not dropping any points.**","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.float_format', lambda x: '%.3f' % x)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:03:05.434943Z","iopub.execute_input":"2024-02-26T18:03:05.435388Z","iopub.status.idle":"2024-02-26T18:03:05.441532Z","shell.execute_reply.started":"2024-02-26T18:03:05.435343Z","shell.execute_reply":"2024-02-26T18:03:05.440122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids.groupby(\"outcome\").mean(numeric_only=True).T","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:03:08.062721Z","iopub.execute_input":"2024-02-26T18:03:08.063160Z","iopub.status.idle":"2024-02-26T18:03:08.085117Z","shell.execute_reply.started":"2024-02-26T18:03:08.063130Z","shell.execute_reply":"2024-02-26T18:03:08.083463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Merchandise concept.\n\nTop merch bought by bots and humans.\n\nTop countries from where bots and humans originate.\n\n**Dont drop the merch column just yet just check if it displays the same pattern for humans and bots if so then dont perform the above operations and use the graph to drop the columns.**","metadata":{}},{"cell_type":"code","source":"bids_data.columns","metadata":{"execution":{"iopub.status.busy":"2024-02-25T19:10:31.035864Z","iopub.execute_input":"2024-02-25T19:10:31.036260Z","iopub.status.idle":"2024-02-25T19:10:31.043514Z","shell.execute_reply.started":"2024-02-25T19:10:31.036232Z","shell.execute_reply":"2024-02-25T19:10:31.042402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids.merge(bids_data, on=\"bidder_id\", how=\"left\").dropna().groupby([\"outcome\", \"merchandise\"])[[\"merchandise\"]].count().rename(columns={'merchandise': 'merch_count'}, errors='raise').sort_values([\"outcome\", \"merch_count\"], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:03:21.396401Z","iopub.execute_input":"2024-02-26T18:03:21.396851Z","iopub.status.idle":"2024-02-26T18:03:36.518988Z","shell.execute_reply.started":"2024-02-26T18:03:21.396819Z","shell.execute_reply":"2024-02-26T18:03:36.517337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = bidder_train_unique_ids.merge(bids_data, on=\"bidder_id\", how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:06:23.844140Z","iopub.execute_input":"2024-02-26T18:06:23.844577Z","iopub.status.idle":"2024-02-26T18:06:32.862636Z","shell.execute_reply.started":"2024-02-26T18:06:23.844547Z","shell.execute_reply":"2024-02-26T18:06:32.861258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids.columns","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:06:36.086385Z","iopub.execute_input":"2024-02-26T18:06:36.086828Z","iopub.status.idle":"2024-02-26T18:06:36.095379Z","shell.execute_reply.started":"2024-02-26T18:06:36.086796Z","shell.execute_reply":"2024-02-26T18:06:36.093940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"robot_filter = temp[\"outcome\"] == 1\nhuman_filter = temp[\"outcome\"] == 0\n\ntemp[robot_filter].groupby(\"country\")[[\"bid_id\"]].count().rename(columns={\"\":\"counts\"}).sort_values(\"bid_id\", ascending=False).head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:06:38.442192Z","iopub.execute_input":"2024-02-26T18:06:38.442623Z","iopub.status.idle":"2024-02-26T18:06:38.673023Z","shell.execute_reply.started":"2024-02-26T18:06:38.442592Z","shell.execute_reply":"2024-02-26T18:06:38.671445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp[human_filter].groupby(\"country\")[[\"bid_id\"]].count().rename(columns={\"\":\"counts\"}).sort_values(\"bid_id\", ascending=False).head(10)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:06:42.512647Z","iopub.execute_input":"2024-02-26T18:06:42.513947Z","iopub.status.idle":"2024-02-26T18:06:44.258869Z","shell.execute_reply.started":"2024-02-26T18:06:42.513892Z","shell.execute_reply":"2024-02-26T18:06:44.257554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance","metadata":{}},{"cell_type":"code","source":"robot_filter = bidder_train_unique_ids[\"outcome\"] == 1\nhuman_filter = bidder_train_unique_ids[\"outcome\"] == 0\n\nbidder_train_unique_ids.columns","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-02-26T18:07:20.856863Z","iopub.execute_input":"2024-02-26T18:07:20.857308Z","iopub.status.idle":"2024-02-26T18:07:20.867366Z","shell.execute_reply.started":"2024-02-26T18:07:20.857275Z","shell.execute_reply":"2024-02-26T18:07:20.865901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"robot_bids = bidder_train_unique_ids[robot_filter]\nhuman_bids = bidder_train_unique_ids[human_filter]","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:07:24.116579Z","iopub.execute_input":"2024-02-26T18:07:24.117000Z","iopub.status.idle":"2024-02-26T18:07:24.126506Z","shell.execute_reply.started":"2024-02-26T18:07:24.116970Z","shell.execute_reply":"2024-02-26T18:07:24.125104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"robot_bids.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:07:30.532306Z","iopub.execute_input":"2024-02-26T18:07:30.532759Z","iopub.status.idle":"2024-02-26T18:07:30.566028Z","shell.execute_reply.started":"2024-02-26T18:07:30.532726Z","shell.execute_reply":"2024-02-26T18:07:30.564519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"robot_bids[\"merch_count\"]","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:07:34.595266Z","iopub.execute_input":"2024-02-26T18:07:34.595743Z","iopub.status.idle":"2024-02-26T18:07:34.606741Z","shell.execute_reply.started":"2024-02-26T18:07:34.595711Z","shell.execute_reply":"2024-02-26T18:07:34.605254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = ['bids_count',\n       'auction_count', 'device_count', 'time', 'country_count',\n       'ip_count', 'url_count', 'count', 'mean', 'std', 'min', '25%', '50%',\n       '75%', 'max', 'iqr', 'total_bids', 'total_second_half_bids',\n       'total_first_half_bids', 'percent_second_half_bids',\n       'percent_first_half_bids', 'num_aucts_with_highest_bids',\n       'max_bids_in_auct', 'num_aucts_with_first_bids',\n       'max_bids_on_any_device', 'max_bids_on_any_device_on_any_auct',\n       'bids_per_auction', 'bids_per_device', 'bids_per_url',\n       'devices_per_auct', 'ip_per_country', 'percent_max_bids']","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:07:36.799487Z","iopub.execute_input":"2024-02-26T18:07:36.799991Z","iopub.status.idle":"2024-02-26T18:07:36.807825Z","shell.execute_reply.started":"2024-02-26T18:07:36.799954Z","shell.execute_reply":"2024-02-26T18:07:36.806232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"robot_bids.columns","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:07:38.948036Z","iopub.execute_input":"2024-02-26T18:07:38.948517Z","iopub.status.idle":"2024-02-26T18:07:38.958595Z","shell.execute_reply.started":"2024-02-26T18:07:38.948483Z","shell.execute_reply":"2024-02-26T18:07:38.956615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nrows = (len(columns) // 2)\nncols = 2\n\ncol_ctr = 0\n\nfig, axes_arr = mp.subplots(nrows, ncols, figsize = (20, 70))\n\nfor i in range(len(columns)):\n#     print(columns[i])\n    _ = sns.kdeplot(data = robot_bids, x = columns[i], ax = axes_arr[i//2, i%2], label=\"Robot\", fill=True, alpha=0.1, color=\"Black\").set_xlim(left=0)\n    _ = sns.kdeplot(data = human_bids, x = columns[i], ax = axes_arr[i//2, i%2], label=\"Human\", fill=True, alpha=0.1, color=\"Red\").set_xlim(left=0)\n    _ = axes_arr[i//2, i%2].legend()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:08:18.623544Z","iopub.execute_input":"2024-02-26T18:08:18.624014Z","iopub.status.idle":"2024-02-26T18:08:30.168874Z","shell.execute_reply.started":"2024-02-26T18:08:18.623982Z","shell.execute_reply":"2024-02-26T18:08:30.167674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Creation","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import GridSearchCV, RepeatedStratifiedKFold\nfrom time import time\nfrom imblearn.over_sampling import RandomOverSampler\nfrom imblearn.pipeline import make_pipeline","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:12:26.108548Z","iopub.execute_input":"2024-02-26T18:12:26.109820Z","iopub.status.idle":"2024-02-26T18:12:27.465509Z","shell.execute_reply.started":"2024-02-26T18:12:26.109775Z","shell.execute_reply":"2024-02-26T18:12:27.463890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bidder_train_unique_ids[\"outcome\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:12:28.928986Z","iopub.execute_input":"2024-02-26T18:12:28.929801Z","iopub.status.idle":"2024-02-26T18:12:28.942635Z","shell.execute_reply.started":"2024-02-26T18:12:28.929758Z","shell.execute_reply":"2024-02-26T18:12:28.941374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Imbalanced Data","metadata":{}},{"cell_type":"code","source":"# X_train.columns\ntemp_cols = ['payment_account', 'address', 'bids_count', 'auction_count',\n       'device_count', 'time', 'country_count', 'ip_count', 'count', 'mean',\n       'std', 'min', '25%', '50%', '75%', 'iqr', 'total_bids',\n       'total_second_half_bids', 'max_bids_in_auct',\n       'num_aucts_with_first_bids', 'max_bids_on_any_device_on_any_auct',\n       'bids_per_auction', 'bids_per_device', 'bids_per_url',\n       'devices_per_auct', 'ip_per_country', \"payment_account\", \"address\"]","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:12:31.947475Z","iopub.execute_input":"2024-02-26T18:12:31.947955Z","iopub.status.idle":"2024-02-26T18:12:31.954726Z","shell.execute_reply.started":"2024-02-26T18:12:31.947920Z","shell.execute_reply":"2024-02-26T18:12:31.953577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Columns to be dropped\n\ncols_to_be_dropped = [\"bidder_id\", \"outcome\",\"max\", \"percent_second_half_bids\", \"percent_first_half_bids\",\n                      \"num_aucts_with_highest_bids\", \"percent_max_bids\",\n                      \"merch_count\", \"total_first_half_bids\", \"url_count\", \"max_bids_on_any_device\", \"payment_account\", \"address\"]\n\nX_train = bidder_train_unique_ids.drop(columns = cols_to_be_dropped, axis = 1)\nY_train = bidder_train_unique_ids[\"outcome\"]","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:12:32.722112Z","iopub.execute_input":"2024-02-26T18:12:32.722775Z","iopub.status.idle":"2024-02-26T18:12:32.730262Z","shell.execute_reply.started":"2024-02-26T18:12:32.722737Z","shell.execute_reply":"2024-02-26T18:12:32.728759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = bidder_test_unique_ids[temp_cols]","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:12:34.290354Z","iopub.execute_input":"2024-02-26T18:12:34.290789Z","iopub.status.idle":"2024-02-26T18:12:34.298390Z","shell.execute_reply.started":"2024-02-26T18:12:34.290757Z","shell.execute_reply":"2024-02-26T18:12:34.297209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfc1 = RandomForestClassifier(random_state = 0)\nrfc2 = RandomForestClassifier(random_state = 1)\nrfc3 = RandomForestClassifier(random_state = 2)\n\nros = RandomOverSampler(sampling_strategy = 0.1, random_state = 123)\n\npp_rfc1 = make_pipeline(ros, rfc1)\npp_rfc2 = make_pipeline(ros, rfc2)\npp_rfc3 = make_pipeline(ros, rfc3)\n\nrfc_base_models = [pp_rfc1, pp_rfc2, pp_rfc3]","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:12:36.978062Z","iopub.execute_input":"2024-02-26T18:12:36.978606Z","iopub.status.idle":"2024-02-26T18:12:36.987384Z","shell.execute_reply.started":"2024-02-26T18:12:36.978566Z","shell.execute_reply":"2024-02-26T18:12:36.985662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cross(base_models, X_train, Y_train):\n    start = time()\n    \n    k_fold_val = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 4, random_state = 5) \n    cv_auc = []\n\n    for train_index, test_index in k_fold_val.split(X_train, Y_train):\n        X_tr, X_ts = X_train.iloc[train_index], X_train.iloc[test_index]\n        Y_tr, Y_ts = Y_train.iloc[train_index], Y_train.iloc[test_index]\n\n        Y_pr = []\n        for model in base_models:\n#             print(Y_tr)\n            model.fit(X_tr, Y_tr)\n            curr_Y_Prob = model.predict_proba(X_ts)[:, 1]\n            Y_pr.append(curr_Y_Prob)\n        Y_pr = np.mean(Y_pr, axis = 0)\n\n        auc = roc_auc_score(Y_ts, Y_pr)\n        cv_auc.append(auc)\n    \n    mean_auc = np.mean(cv_auc)\n    end = time()\n\n    print(f\"Time elapsed: {(end - start):.4f} seconds\")\n    print(f\"AUC:{mean_auc:.4f}\")\n    \n    return cv_auc","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:12:40.726793Z","iopub.execute_input":"2024-02-26T18:12:40.727233Z","iopub.status.idle":"2024-02-26T18:12:40.738186Z","shell.execute_reply.started":"2024-02-26T18:12:40.727198Z","shell.execute_reply":"2024-02-26T18:12:40.736275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_indi_rf_AUCs = cross([rfc_base_models[2]], X_train, Y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:12:43.007225Z","iopub.execute_input":"2024-02-26T18:12:43.008171Z","iopub.status.idle":"2024-02-26T18:13:03.384991Z","shell.execute_reply.started":"2024-02-26T18:12:43.008132Z","shell.execute_reply":"2024-02-26T18:13:03.383474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_avg_rf_AUCs = cross(rfc_base_models, X_train, Y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:13:03.387551Z","iopub.execute_input":"2024-02-26T18:13:03.387936Z","iopub.status.idle":"2024-02-26T18:14:04.457972Z","shell.execute_reply.started":"2024-02-26T18:13:03.387907Z","shell.execute_reply":"2024-02-26T18:14:04.456580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_avg_rf_AUCs","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:14:04.459711Z","iopub.execute_input":"2024-02-26T18:14:04.460064Z","iopub.status.idle":"2024-02-26T18:14:04.468870Z","shell.execute_reply.started":"2024-02-26T18:14:04.460037Z","shell.execute_reply":"2024-02-26T18:14:04.467517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Grid Search CV","metadata":{}},{"cell_type":"code","source":"hyper_grid = {'randomforestclassifier__n_estimators': [100, 200, 300, 400],\n        'randomforestclassifier__max_depth': [None, 5, 8, 10],\n        'randomforestclassifier__min_samples_split': [2, 5, 10],\n        'randomforestclassifier__min_samples_leaf': [1, 2, 4],\n       }","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:14:41.782436Z","iopub.execute_input":"2024-02-26T18:14:41.782925Z","iopub.status.idle":"2024-02-26T18:14:41.791448Z","shell.execute_reply.started":"2024-02-26T18:14:41.782891Z","shell.execute_reply":"2024-02-26T18:14:41.789445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time()\n\nbest_rf_models = []\n\nfor model in rfc_base_models:\n\n    search_model = GridSearchCV(estimator = model,\n                          param_grid = hyper_grid,\n                          scoring = 'roc_auc', \n                          cv = 3,)\n    search_model.fit(X_train,Y_train)\n    best_rf_models.append(search_model.best_estimator_)\n\nend = time()\n\nprint(f\"Time Elapsed: {(end - start):.4f} seconds\")","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:14:43.782172Z","iopub.execute_input":"2024-02-26T18:14:43.782711Z","iopub.status.idle":"2024-02-26T18:37:04.785231Z","shell.execute_reply.started":"2024-02-26T18:14:43.782673Z","shell.execute_reply":"2024-02-26T18:37:04.782441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(best_rf_models)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:37:04.790392Z","iopub.execute_input":"2024-02-26T18:37:04.790967Z","iopub.status.idle":"2024-02-26T18:37:04.817038Z","shell.execute_reply.started":"2024-02-26T18:37:04.790913Z","shell.execute_reply":"2024-02-26T18:37:04.815513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"individual_rf_AUCs = cross([best_rf_models[0]], X_train, Y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:37:34.199978Z","iopub.execute_input":"2024-02-26T18:37:34.200393Z","iopub.status.idle":"2024-02-26T18:37:53.293334Z","shell.execute_reply.started":"2024-02-26T18:37:34.200362Z","shell.execute_reply":"2024-02-26T18:37:53.291745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"average_rf_AUCs =cross(best_rf_models, X_train, Y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:37:53.295573Z","iopub.execute_input":"2024-02-26T18:37:53.295943Z","iopub.status.idle":"2024-02-26T18:40:11.613181Z","shell.execute_reply.started":"2024-02-26T18:37:53.295912Z","shell.execute_reply":"2024-02-26T18:40:11.611406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtc1 = DecisionTreeClassifier(random_state = 0)\ndtc2 = DecisionTreeClassifier(random_state = 1)\ndtc3 = DecisionTreeClassifier(random_state = 2)\n\nros = RandomOverSampler(sampling_strategy = 0.1, random_state = 123)\n\npp_dtc1 = make_pipeline(ros, dtc1)\npp_dtc2 = make_pipeline(ros, dtc2)\npp_dtc3 = make_pipeline(ros, dtc3)\n\ndtc_base_models = [pp_dtc1, pp_dtc2, pp_dtc3]","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:40:56.148491Z","iopub.execute_input":"2024-02-26T18:40:56.149014Z","iopub.status.idle":"2024-02-26T18:40:56.158542Z","shell.execute_reply.started":"2024-02-26T18:40:56.148982Z","shell.execute_reply":"2024-02-26T18:40:56.156591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_indi_dt_AUCs = cross([dtc_base_models[2]], X_train, Y_train)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:40:56.705994Z","iopub.execute_input":"2024-02-26T18:40:56.706971Z","iopub.status.idle":"2024-02-26T18:40:58.291724Z","shell.execute_reply.started":"2024-02-26T18:40:56.706921Z","shell.execute_reply":"2024-02-26T18:40:58.289984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyper_dt_grid = {'decisiontreeclassifier__max_depth': [None, 5, 8, 10],\n        'decisiontreeclassifier__min_samples_split': [2, 5, 10],\n        'decisiontreeclassifier__min_samples_leaf': [1, 2, 4],\n       }","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:40:58.294695Z","iopub.execute_input":"2024-02-26T18:40:58.295101Z","iopub.status.idle":"2024-02-26T18:40:58.302091Z","shell.execute_reply.started":"2024-02-26T18:40:58.295068Z","shell.execute_reply":"2024-02-26T18:40:58.300415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time()\n\nnow = datetime.datetime.now()\nprint(\"Started At - \", now.strftime(\"%Y-%m-%d %H:%M:%S\"))\n\nbest_dtc_models = []\n\nfor model in dtc_base_models:\n\n    search_model = GridSearchCV(estimator = model,\n                          param_grid = hyper_dt_grid,\n                          scoring = 'roc_auc', \n                          cv = 3,)\n    search_model.fit(X_train,Y_train)\n    best_dtc_models.append(search_model.best_estimator_)\n\nend = time()\n\nnow = datetime.datetime.now()\nprint (\"Ended At - \", now.strftime(\"%Y-%m-%d %H:%M:%S\"))\n\nprint(f\"Time Elapsed: {(end - start):.4f} seconds\")","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:40:58.304297Z","iopub.execute_input":"2024-02-26T18:40:58.304860Z","iopub.status.idle":"2024-02-26T18:41:10.329276Z","shell.execute_reply.started":"2024-02-26T18:40:58.304825Z","shell.execute_reply":"2024-02-26T18:41:10.326757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_AUCs_list = [('Base individual RF', base_indi_rf_AUCs), ('Base average RFs', base_avg_rf_AUCs),\n                   ('Tuned individual RF', individual_rf_AUCs), ('Tuned average RFs', average_rf_AUCs)]\n\nfig, ax = mp.subplots(figsize = (20, 10))\n\nfor i in range(len(model_AUCs_list)):\n    color = next(ax._get_lines.prop_cycler)['color']\n    \n    AUC_distribution = sns.kdeplot(x = model_AUCs_list[i][1],\n                                   ax = ax,\n                                   label = model_AUCs_list[i][0],\n                                   color = color)\n\n    x_coord = ax.lines[-1].get_xdata()     # Gets the all the X points of the current line plotted\n    y_coord = ax.lines[-1].get_ydata()     # Gets the all the Y points of the current line plotted\n    index_of_max_y = np.argmax(y_coord)    # Finds the index of the highest Y Co-ordinate.\n    ax.axvline(x_coord[index_of_max_y], linestyle = '--', linewidth = 1, color = color)    # Uses that index to plot a straight line on the respective X Co-ord.\n\nax.legend(loc = 'upper left', fontsize = 'x-large')\nmp.title('ROC AUC score distribution across 3 RFC models', fontsize = 'xx-large')\nmp.xlabel('ROC AUC score', fontsize = 'x-large')\nmp.ylabel('Density', fontsize = 'x-large')\nmp.xlim(right = 1)\n\nmp.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:50:56.313580Z","iopub.execute_input":"2024-02-26T18:50:56.314154Z","iopub.status.idle":"2024-02-26T18:50:56.943228Z","shell.execute_reply.started":"2024-02-26T18:50:56.314118Z","shell.execute_reply":"2024-02-26T18:50:56.942072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission via the Final Model","metadata":{}},{"cell_type":"code","source":"final_rf_models = best_rf_models","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:51:03.559838Z","iopub.execute_input":"2024-02-26T18:51:03.560368Z","iopub.status.idle":"2024-02-26T18:51:03.566705Z","shell.execute_reply.started":"2024-02-26T18:51:03.560303Z","shell.execute_reply":"2024-02-26T18:51:03.565195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:51:04.245935Z","iopub.execute_input":"2024-02-26T18:51:04.246514Z","iopub.status.idle":"2024-02-26T18:51:04.273549Z","shell.execute_reply.started":"2024-02-26T18:51:04.246472Z","shell.execute_reply":"2024-02-26T18:51:04.272188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.drop(['payment_account', 'address'], axis = 1, inplace = True)\nX_test.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:51:04.734890Z","iopub.execute_input":"2024-02-26T18:51:04.735441Z","iopub.status.idle":"2024-02-26T18:51:04.770603Z","shell.execute_reply.started":"2024-02-26T18:51:04.735400Z","shell.execute_reply":"2024-02-26T18:51:04.769348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prob_rf = []\n\nfor model in final_rf_models:\n    model.fit(X_train, Y_train)\n\n    prob_rf_vals = model.predict_proba(X_test)[:,1]\n    prob_rf.append(prob_rf_vals)\n\nresult = np.mean(prob_rf, axis = 0)\nresult","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:51:07.344699Z","iopub.execute_input":"2024-02-26T18:51:07.345178Z","iopub.status.idle":"2024-02-26T18:51:11.370970Z","shell.execute_reply.started":"2024-02-26T18:51:07.345143Z","shell.execute_reply":"2024-02-26T18:51:11.369394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final CSV for Submission","metadata":{}},{"cell_type":"code","source":"final_output = pd.DataFrame({\n    'bidder_id': bidder_test_unique_ids['bidder_id'],\n    'prediction': result\n})\nfinal_output.to_csv('output.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-26T18:51:40.284362Z","iopub.execute_input":"2024-02-26T18:51:40.285664Z","iopub.status.idle":"2024-02-26T18:51:40.320853Z","shell.execute_reply.started":"2024-02-26T18:51:40.285610Z","shell.execute_reply":"2024-02-26T18:51:40.319011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}