{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4294,"databundleVersionId":33829,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%bash\nPYTHONPATH=\nmkdir -p ~/miniconda3\nwget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh\nbash ~/miniconda3/miniconda.sh -b -u -p /usr/local\nrm ~/miniconda3/miniconda.sh","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T15:51:17.369123Z","iopub.execute_input":"2025-07-22T15:51:17.369533Z","iopub.status.idle":"2025-07-22T15:51:55.626630Z","shell.execute_reply.started":"2025-07-22T15:51:17.369499Z","shell.execute_reply":"2025-07-22T15:51:55.625647Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!which conda\n!echo $PATH","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T15:54:09.406060Z","iopub.execute_input":"2025-07-22T15:54:09.406451Z","iopub.status.idle":"2025-07-22T15:54:09.648939Z","shell.execute_reply.started":"2025-07-22T15:54:09.406410Z","shell.execute_reply":"2025-07-22T15:54:09.647497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\nconda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main\nconda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T15:57:16.261992Z","iopub.execute_input":"2025-07-22T15:57:16.262940Z","iopub.status.idle":"2025-07-22T15:57:17.997087Z","shell.execute_reply.started":"2025-07-22T15:57:16.262900Z","shell.execute_reply":"2025-07-22T15:57:17.996294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!conda create -y -q --name py27 python=2.7","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T15:59:32.366959Z","iopub.execute_input":"2025-07-22T15:59:32.367369Z","iopub.status.idle":"2025-07-22T15:59:39.396121Z","shell.execute_reply.started":"2025-07-22T15:59:32.367337Z","shell.execute_reply":"2025-07-22T15:59:39.394685Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!source activate py27 && python -m pip install pandas==0.18.0 numpy scikit-learn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T16:45:40.472970Z","iopub.execute_input":"2025-07-22T16:45:40.473432Z","iopub.status.idle":"2025-07-22T16:51:12.728654Z","shell.execute_reply.started":"2025-07-22T16:45:40.473393Z","shell.execute_reply":"2025-07-22T16:51:12.727073Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!find /kaggle/input -name \"*.zip\" -exec unzip {} -d /kaggle/working/extracted \\;\n!ls /kaggle/working/extracted ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T17:06:34.428503Z","iopub.execute_input":"2025-07-22T17:06:34.428899Z","iopub.status.idle":"2025-07-22T17:06:44.373899Z","shell.execute_reply.started":"2025-07-22T17:06:34.428869Z","shell.execute_reply":"2025-07-22T17:06:44.372326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\nsource activate py27\n\npython\n'''\nREADME\n\nhttps://www.kaggle.com/c/facebook-recruiting-iv-human-or-bot/\nThis script classifies bidders in an online auction as either robot or human\n\nTo run the code \nbids, bot_or_human = load()\nX = build_X(bids, bot_or_human)\nmake(X)\n'''\n\n\nimport pandas as pd\nimport numpy as np\n#from matplotlib import pyplot as plt\n\nimport sklearn.preprocessing as preprocessing\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.linear_model import LogisticRegression, BayesianRidge\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.metrics import roc_auc_score\n\n#chan\nimport pickle\n'''\nDeclare time units\n'''\none_day = 4547368124071.8799\nstartt = 9631916820392676.0\ndelta = 2.16e10\ncutoff = 1.5e8 #cutoff for \"small\" time scales\n\n'''\nHow to calculate time units\nstart and end times for each three-day data segment\n-9631916842105263 + 9645558894736842 = 13642052631579  #week 0\n-9695580000000000 + 9709222052631578 = 13642052631578  #week 2\n-9759243157894736 + 9772885210526315 = 13642052631579  #week 4\ngap between end week 0, start week 2: 50021105263158\ngap between end week 2, start week 4: 50021105263158\n\n#use linear regression to calculation startt and one_day\nfrom numpy import arange,array,ones,linalg\ntimes = np.array([9631916842105263, 9645558894736842+1, 9695580000000000, 9709222052631578+1, 9759243157894736, 9772885210526315+1])\ndays = np.array([0, 3, 14, 17, 28, 31]).reshape((-1,1))\nfrom sklearn import linear_model\nclf = linear_model.LinearRegression(fit_intercept=True)\nclf.fit(days, times)\n4547368124071.8799\nstartt = clf.predict([0])\nstartt = 9631916820392676.0\n\n'''\n\n#bids, bot_or_human = load()\ndef load():\n    bot_or_human = pd.read_csv('extracted/train.csv')\n    test = pd.read_csv('extracted/test.csv')\n    test['outcome'] = -1\n    \n    \n    bids = pd.read_csv('extracted/bids.csv')\n    bids = bids.sort(['auction', 'time'])\n    \n    bot_or_human = pd.concat([ bot_or_human, test ])\n    \n    #strip the last five characters from the address and payment accounts\n    bot_or_human.address = bot_or_human.address.apply(address_strip)\n    bot_or_human.payment_account = bot_or_human.payment_account.apply(address_strip)\n    \n    \n    '''\n    pivot bidder address prefix and bidder payment_account prefix into features\n    '''\n    bot_or_human['payment_account_prefix_same_as_address_prefix'] = bot_or_human['address']==bot_or_human['payment_account']\n    max_relevant = 6\n    \n    b = bot_or_human[['bidder_id', 'address']].groupby('address').size().reset_index().sort(0, ascending=False)\n    b = b.rename(columns = {0:'address_count'})\n    #repeat_addresses = b[b.address_count >= max_relevant].address.values\n    infrequent_addresses = b[b.address_count >= 2][b.address_count < max_relevant].address.values\n    rare_addresses = b[b.address_count < 2].address.values\n    bot_or_human['address'].loc[bot_or_human.address.isin(rare_addresses)] = 'rare_address'\n    bot_or_human['address'].loc[bot_or_human.address.isin(infrequent_addresses)] = 'infrequent_address'\n    #b = bot_or_human[['bidder_id', 'address']].pivot(index='bidder_id',columns='address').fillna(0).reset_index()\n    b = pd.get_dummies(bot_or_human[['address']])\n    bot_or_human = pd.concat([bot_or_human, b], axis=1)\n    \n    b = bot_or_human[['bidder_id', 'payment_account']].groupby('payment_account').size().reset_index().sort(0, ascending=False)\n    b = b.rename(columns = {0:'payment_account_count'})\n    #repeat_payment_accounts = b[b.payment_account_count >= max_relevant].payment_account.values\n    infrequent_payment_account = b[b.payment_account_count >= 2][b.payment_account_count < max_relevant].payment_account.values\n    rare_payment_account = b[b.payment_account_count < 2].payment_account.values\n    bot_or_human['payment_account'].loc[bot_or_human.payment_account.isin(rare_payment_account)] = 'rare_account'\n    bot_or_human['payment_account'].loc[bot_or_human.payment_account.isin(infrequent_payment_account)] = 'infrequent_account'    \n    #bot_or_human['payment_account'].loc[~bot_or_human.payment_account.isin(repeat_payment_accounts)] = 'only_once'\n    b = pd.get_dummies(bot_or_human[['payment_account']])\n    bot_or_human = pd.concat([bot_or_human, b], axis=1)\n\n        \n    return bids, bot_or_human\n    \n    \ndef address_strip(x):\n    return x[0:-5]    \n    \n\n\n\ndef dt(X, bids):\n    bots = X[X.outcome==1].bidder_id.values\n    humans = X[X.outcome==0].bidder_id.values\n\n    '''\n    auction duration\n    check if auctions run longer than three days\n    calculate time from bid until time of last bid in the auction\n    calculate time from bid until time of first bid in the auction\n    '''    \n    times = bids.groupby('auction').time.min().reset_index()\n    times=times.rename(columns = {'time':'startt'})\n    times2 = bids.groupby('auction').time.max().reset_index()\n    times2=times2.rename(columns = {'time':'endt'})\n    \n    times = pd.merge(times, times2, on='auction', how='left')\n    times['duration'] = times.endt - times.startt\n    times['short'] = 1.0*(times['duration'] < 3.01*one_day)\n    \n    bids = pd.merge(bids, times[['auction', 'short', 'startt', 'endt']], on='auction', how='left')\n    b = bids.groupby('bidder_id').short.mean().reset_index()\n    X = pd.merge(X, b, on='bidder_id', how='left')\n    \n    #time from bid until end of auction\n    bids['t_until_end'] = bids.endt - bids.time\n    bids['t_since_start'] = bids.time - bids.startt\n    \n    b = bids.groupby('bidder_id').t_until_end.median().reset_index()\n    b=b.rename(columns = {'t_until_end':'t_until_end_median'})\n    X = pd.merge(X, b, on='bidder_id', how='left')    \n\n    b = bids.groupby('bidder_id').t_since_start.median().reset_index()\n    b=b.rename(columns = {'t_since_start':'t_since_start_median'})\n    X = pd.merge(X, b, on='bidder_id', how='left')        \n\n\n    '''\n    Useful plots\n    \n    test_ids = X[X.outcome==-1].bidder_id.values\n    \n    bot = bids[bids.bidder_id.isin(bots)][bids.country.isin(['de', 'nl', 'se', 'no', 'cz', 'hu'])].t_until_end\n    human = bids[bids.bidder_id.isin(humans)][bids.country.isin(['de', 'nl', 'se', 'no', 'cz', 'hu'])].t_until_end\n\n    bot = bids[bids.bidder_id.isin(bots)][bids.country.isin(['id'])].t_until_end\n    human = bids[bids.bidder_id.isin(humans)][bids.country.isin(['id'])].t_until_end\n\n    bot = bids[bids.bidder_id.isin(bots)].t_until_end\n    human = bids[bids.bidder_id.isin(humans)].t_until_end\n    botp = np.histogram(bot, 800, range=[0,8e13], normed=True)\n    humanp = np.histogram(human, 800, range=[0,8e13], normed=True)\n\n    bot = np.mod(bids[bids.bidder_id.isin(bots)].t_until_end.values, 4.55e12)\n    human = np.mod(bids[bids.bidder_id.isin(humans)].t_until_end.values, 4.55e12)\n            \n    botp = np.histogram(bot, 300, range=[0,4.9e12], normed=True)\n    humanp = np.histogram(human, 300, range=[0,4.9e12], normed=True)\n    \n    plt.plot(botp[1][0:-1], botp[0], 'b.-')\n    plt.plot(botp[1][0:-1], humanp[0], 'g.-')\n\n\n\n    bot = bids[bids.bidder_id.isin(bots)].time\n    human = bids[bids.bidder_id.isin(humans)].time\n    test = bids[bids.bidder_id.isin(test_ids)].time\n        \n    bot = bids[bids.bidder_id.isin(bots)][bids.country.isin(['nl'])].time\n    human = bids[bids.bidder_id.isin(humans)][bids.country.isin(['nl'])].time    \n\n    bots = ['5354c02817e47f28e60e44a40ca5e48dll4sz']\n    humans = ['ff58ffde976a4899dcd89597a7877e18lntgz']\n    \n    bot = np.mod(bids[bids.bidder_id.isin(bots)].time.values, one_day)\n    human = np.mod(bids[bids.bidder_id.isin(humans)].time.values, one_day)\n    humanp = np.histogram(human, 48, range=[0, one_day], normed=True)\n    botp = np.histogram(bot, 48, range=[0, one_day], normed=True)\n\n\n    humanp = np.histogram(human, 4000, range=[9.63e15, 9.78e15], normed=True)\n    botp = np.histogram(bot, 4000, range=[9.63e15, 9.78e15], normed=True)\n    testp = np.histogram(test, 4000, range=[9.63e15, 9.78e15], normed=True)    \n        \n    humanp = np.histogram(human, 30, range=[9.63e15, 9.9e15], normed=True)\n    botp = np.histogram(bot, 30, range=[9.63e15, 9.9e15], normed=True)\n    \n    \n    humanp = np.histogram(human, 1000, range=[9.631e15, 9.645e15], normed=True)\n    botp = np.histogram(bot, 1000, range=[9.631e15, 9.645e15], normed=True)\n        \n    plt.plot(humanp[1][0:-1], testp[0], 'r.-')    \n    plt.plot(humanp[1][0:-1], humanp[0], 'g.-')\n    plt.plot(humanp[1][0:-1], botp[0], 'b.-')\n    plt.ylabel('fraction of bids made between t and t+dt')\n    plt.xlabel('time [arbitrary units]')\n    plt.legend(['human','bot'], frameon=False)\n    plt.title('bids over time')\n    plt.show()\n    \n    dt = humanp[1][1] - humanp[1][0]\n    a = np.correlate(humanp[0], humanp[0], mode='full')\n    plt.plot(dt*np.array(range(0,len(a)-len(a)/2)), a[len(a)/2:], 'k.-')\n    plt.ylabel('autocorrelation')\n    plt.xlabel('time')\n    plt.title('autocorrelation of bids/unit time histogram')\n    \n    #sample bids per unit time over an auction\n    plt.hist(bids[bids.auction == bids.auction[3]].time.values, bins=200)\n    plt.ylabel('bids')\n    plt.xlabel('time')\n    plt.title('bids during auction')\n    '''\n\n    \n    '''\n    number of bids per hour (n=24), half hour (n=48)\n    '''\n    times = [72]\n    for n in times: \n        print 'hour ', n\n        b = np.floor((1.0*n/one_day*np.mod(bids.time-startt-delta, one_day)))\n        b.name = 'hour'+str(n)\n        bids = pd.concat([bids, b], axis=1)\n        b = bids.groupby(['bidder_id', 'hour'+str(n)]).size().reset_index()\n        b=b.rename(columns = {0:'bids_in_hour'})\n        b=b.pivot('bidder_id', 'hour'+str(n), 'bids_in_hour').fillna(0).reset_index()\n        old_names = b.columns.values\n        new_names = b.columns.values\n        for i in range(1,len(b.columns)):\n            new_names[i] = str(int(old_names[i])) + '_hour'+str(n)\n        b = b.rename(columns=dict(zip(old_names, new_names)))\n        b['max_bids_in_hour'+str(n)] = b.iloc[:, 1:].max(axis=1)\n    \n        for i in range(n):\n            b[str(i)+'_hour'+str(n)] = b[str(i)+'_hour'+str(n)]/b['max_bids_in_hour'+str(n)]\n        #b = b.drop('max_bids_in_hour'+str(n), 1)\n        X = pd.merge(X, b, on='bidder_id', how='left')\n                \n    n = times[0]\n    X['sleep'] = 0\n    step = int(5.5*n/24)\n    for i in xrange(n - step):\n        X['sleep'] += X.loc[:, str(i)+'_hour'+str(n):str(i+step)+'_hour'+str(n)].sum(axis=1) == 0\n    \n    X['sleep'] = (X['sleep'] >= 1)    \n    \n        \n    '''\n    dt between a user's bid and the previous bid by another user in the same auction\n    '''\n    for week in [0, 2, 4]:\n        b = bids[bids.week==week].sort(['auction', 'time'])[['auction', 'time', 'bidder_id']]\n        b['bidder_id_prev'] = pd.Series(np.append([np.nan], b.bidder_id.values[0:-1]), index=b.index)\n        b['auction_prev'] = pd.Series(np.append([np.nan], b.auction.values[0:-1]), index=b.index)\n        dt = np.append([np.nan], b.time.values[0:-1])\n        b['time_prev'] = pd.Series(dt, index=b.index)\n        b['dt'] = b['time'] - b['time_prev']\n                \n        if week==0:\n            c = [b[b.bidder_id != b.bidder_id_prev][b.auction == b.auction_prev]]\n        else:\n            c.append(b[b.bidder_id != b.bidder_id_prev][b.auction == b.auction_prev])\n\n    c = pd.concat(c)    \n    d = c.groupby('bidder_id').dt.median().reset_index()\n    d = d.rename(columns = {'dt': 'dt_others_median'})\n    X = pd.merge(X, d[['bidder_id', 'dt_others_median']], on='bidder_id', how='left')\n    #X.loc[:,'dt_others_median'] = X.loc[:,'dt_others_median'].fillna(3.0*one_day)\n    \n    c['dt_others_lt_cutoff'] = np.nan\n    c.loc[c.dt <= cutoff, 'dt_others_lt_cutoff'] = 1.0\n    c.loc[c.dt > cutoff, 'dt_others_lt_cutoff'] = 0.0\n    \n    a= c[['bidder_id', 'dt_others_lt_cutoff']].dropna().groupby('bidder_id').mean().reset_index()\n    X = pd.merge(X, a, on='bidder_id', how='left')\n    X=X.rename(columns = {'dt_others_lt_cutoff':'f_dt_others_lt_cutoff'})    \n    \n    #plt.hist(X[X.outcome==0].dt_median.values, bins=100, range=[0, 1e9], normed=True)\n    #plt.hist(X[X.outcome==1].dt_median.values, bins=100, range=[0, 1e9], normed=True)\n    \n        \n             \n    #time between bids by same bidder (does not matter if previous bid is in same auction)\n    for week in [0, 2, 4]:\n        b = bids[bids.week==week].sort(['bidder_id', 'time'])[['bidder_id', 'time']]\n        #b['bidder_id_prev'] = np.nan\n        #b.bidder_id_prev[1:] = b.bidder_id[0:-1]\n        b['bidder_id_prev'] = pd.Series(np.append([np.nan], b.bidder_id.values[0:-1]), index=b.index)\n        \n        #b['time_prev'] = np.nan\n        dt = np.append([np.nan], b.time.values[0:-1])\n        b['time_prev'] = pd.Series(dt, index=b.index)\n        b['dt'] = b['time'] - b['time_prev']\n        \n        \n        if week==0:\n            c = [b[b.bidder_id == b.bidder_id_prev]]\n        else:\n            c.append(b[b.bidder_id == b.bidder_id_prev])    \n    \n    c = pd.concat(c)    \n    d = c.groupby('bidder_id').dt.median().reset_index()\n    d = d.rename(columns = {'dt': 'dt_self_median'})\n    X = pd.merge(X, d[['bidder_id', 'dt_self_median']], on='bidder_id', how='left')\n    d = c.groupby('bidder_id').dt.min().reset_index()\n    d = d.rename(columns = {'dt': 'dt_self_min'})\n    X = pd.merge(X, d[['bidder_id', 'dt_self_min']], on='bidder_id', how='left')\n    #X.loc[:,'dt_self_median'] = X.loc[:,'dt_self_median'].fillna(3.0*one_day)\n    #X.loc[:,'dt_self_min'] = X.loc[:,'dt_self_min'].fillna(3.0*one_day)\n\n    \n    '''\n    bot_h = np.histogram(bids[bids.bidder_id.isin(bots)].dt.values, bins=100, range=[0, 1e10], normed=True)\n    human_h = np.histogram(bids[bids.bidder_id.isin(humans)].dt.values, bins=100, range=[0, 1e10], normed=True)\n    plt.plot(bot_h[1][1:], bot_h[0], 'b.-')\n    plt.plot(bot_h[1][1:], human_h[0], 'r.-')\n    '''    \n\n\n        \n    return X\n\n    \n    \n\ndef n_bids(X, bids):\n    a = bids.groupby(['bidder_id', 'auction']).size().reset_index()\n    a=a.rename(columns = {0:'bids_per_auction'})\n    b = a.groupby('bidder_id').bids_per_auction.median().reset_index()\n    b=b.rename(columns = {'bids_per_auction':'bids_per_auction_median'})\n    X = pd.merge(X, b, on='bidder_id', how='left')\n    \n    b = a.groupby('bidder_id').mean().reset_index()\n    b=b.rename(columns = {'bids_per_auction':'bids_per_auction_mean'})\n    X = pd.merge(X, b, on='bidder_id', how='left')    \n    \n    b = bids.groupby('bidder_id').size().reset_index()\n    b=b.rename(columns = {0:'n_bids'})\n    X = pd.merge(X, b, on='bidder_id', how='left')    \n        \n    return X\n\n\ndef enumerate(x):\n    x['bid_order'] = 1.0*np.arange(len(x))/len(x)\n    return x\n  \n  \ndef fac_div_fac(x,y):\n\n    if x == 0:\n        x = 1\n    if y == 0:\n        y = 1\n            \n    if x > y:\n        return np.sum( np.log(1.0*np.array(range(int(y+1),int(x+1))) ) )\n    if y > x:\n        return -np.sum( np.log(1.0*np.array(range(int(x+1),int(y+1))) ) )\n    return 0.0\n        \n  \ndef balance(x):\n#    x['balance'] = np.std(x)/np.sqrt(np.mean(x))\n    \n    x0 = int(np.floor(np.sum(x)/len(x)))\n    base = x0*np.ones(len(x))\n    base[0:int(np.sum(x)-x0*len(x))] = x0+1\n    \n    b = 0.0\n    for i in xrange(len(x)):\n        #print base[i], x[i]\n        b += fac_div_fac(base[i], x[i])\n    \n    #print b\n    x['balance'] = b\n    \n    return x  \n\n\n\n\ndef day(X, bids):\n    bots = X[X.outcome==1].bidder_id.values\n    humans = X[X.outcome==0].bidder_id.values\n\n    bids['day'] = np.floor((bids['time'] - startt)/one_day)\n    \n    b = bids.groupby(['bidder_id', 'day']).size().reset_index()\n    b = b.rename(columns = {0:'daily_bids'})\n    b = b.pivot('bidder_id', 'day', 'daily_bids').fillna(0).reset_index()\n    old_names = b.columns.values\n    new_names = [b.columns[0]]    \n    for i in range(1,len(b.columns)):\n        new_names.append( str(int(old_names[i])) + '_day' )\n    b = b.rename(columns=dict(zip(old_names, new_names)))    \n    \n    \n    b['monday'] = (b['0_day'] + b['14_day'] + b['28_day']).fillna(0)\n    b['tuesday'] = (b['1_day'] + b['15_day'] + b['29_day']).fillna(0)\n    b['wednesday'] = (b['2_day'] + b['16_day'] + b['30_day']).fillna(0)\n    b['balance'] = b[['monday', 'tuesday', 'wednesday']].apply(balance, axis=1)[['balance']]\n    b['balance2'] = b[['0_day', '1_day', '2_day', '14_day', '15_day', '16_day', '28_day', '29_day', '30_day']].apply(balance, axis=1)[['balance']]\n    \n    X = pd.merge(X, b.loc[:,['bidder_id', 'monday', 'tuesday', 'wednesday', 'balance', 'balance2']], on='bidder_id', how='left')\n    X[['monday', 'tuesday', 'wednesday', 'balance']] = X[['monday', 'tuesday', 'wednesday', 'balance']].fillna(0)\n\n    s = X.loc[:, ['monday', 'tuesday', 'wednesday']].max(axis=1)\n    X['s_monday'] = X['monday']/s\n    X['s_tuesday'] = X['tuesday']/s\n    X['s_wednesday'] = X['wednesday']/s\n    \n    s = X.loc[:, ['monday', 'tuesday', 'wednesday']].sum(axis=1)\n    X['f_monday'] = X['monday']/s\n    X['f_tuesday'] = X['tuesday']/s\n    X['f_wednesday'] = X['wednesday']/s    \n    \n#     nbids = X.loc[:, ['monday', 'tuesday', 'wednesday']].sum(axis=1)\n#     X['bids_week0'] = (b['0_day'] + b['1_day'] + b['2_day']).fillna(0)\n#     X['bids_week2'] = (b['14_day'] + b['15_day'] + b['16_day']).fillna(0)\n#     X['bids_week4'] = (b['28_day'] + b['29_day'] + b['30_day']).fillna(0)\n#     s = X.loc[:, ['bids_week0', 'bids_week2', 'bids_week4']].max(axis=1)\n#     X['s_bids_week0'] = X['bids_week0']/s\n#     X['s_bids_weeek2'] = X['bids_week2']/s\n#     X['s_bids_week4'] = X['bids_week4']/s\n\n    X.drop('balance2', 1, inplace=True)\n\n    return X\n     \n     \n     \n        \ndef bid_order(X, bids):\n    bids = bids.groupby('auction').apply(enumerate)\n    \n    a = bids.sort('time', ascending=True).groupby('auction', as_index=False).first()\n    b = a.groupby('bidder_id').size().reset_index()\n    b= b.rename(columns = {0:'num_first_bid'})    \n    X = pd.merge(X, b, on='bidder_id', how='left')\n    print X.columns\n    X['num_first_bid'].fillna(0, inplace=True)\n    \n\n    return X\n\n\n\n\ndef user_countries_per_auction(X, bids):\n    #a = bids.groupby(['auction', 'bidder_id', 'country']).country.count()\n    #a = bids.groupby(['auction', 'bidder_id']).country.size()\n    #bids[bids.bidder_id=='5c28b04424e96267711d5af2db62e022tp36c'][['auction', 'country']].groupby('auction').size()\n    \n    \n    b = bids[['bidder_id', 'auction', 'country']].groupby(['bidder_id','auction']).country.nunique().reset_index()\n    b= b.rename(columns = {'country':'countries_per_bidder_per_auction'})    \n    \n    c = b.groupby('bidder_id').countries_per_bidder_per_auction.median().reset_index()\n    c= c.rename(columns = {'countries_per_bidder_per_auction':'countries_per_bidder_per_auction_median'})\n    X = pd.merge(X, c, on='bidder_id', how='left')\n\n    c = b.groupby('bidder_id').countries_per_bidder_per_auction.mean().reset_index()\n    c= c.rename(columns = {'countries_per_bidder_per_auction':'countries_per_bidder_per_auction_mean'})\n    X = pd.merge(X, c, on='bidder_id', how='left')    \n    \n    c = b.groupby('bidder_id').countries_per_bidder_per_auction.max().reset_index()\n    c= c.rename(columns = {'countries_per_bidder_per_auction':'countries_per_bidder_per_auction_max'})\n    X = pd.merge(X, c, on='bidder_id', how='left')        \n    \n    #chan    \n    with open('bidsX.pkl', 'wb') as f: # 'wb' for write binary mode\n        pickle.dump([bids, X], f)\n    \n        \n    a = bids.groupby(['bidder_id']).country.value_counts().reset_index()\n    b = a.groupby(['bidder_id']).agg(lambda x: x.iloc[0]).reset_index()\n    b= b.rename(columns = {'country':'most_common_country'})\n    X = pd.merge(X, b[['bidder_id','most_common_country']], on='bidder_id', how='left')    \n\n    \n    b = bids[['bidder_id', 'country']].groupby(['bidder_id', 'country']).size().reset_index()\n    b= b.rename(columns = {0:'nbids'})\n    c = b.pivot('bidder_id','country').fillna(0).reset_index()\n    c.columns = c.columns.get_level_values(1)\n    c= c.rename(columns={c.columns[0]:'bidder_id'})\n    #fraction of bids in each country (for each user)\n    #c.iloc[:,1:] = c.iloc[:,1:].div(c.iloc[:,1:].sum(axis=1), axis=0)\n    #number of bids in each country rescaled by the largest number of bids in a country (for each user)\n    c.iloc[:,1:] = c.iloc[:,1:].div(c.iloc[:,1:].max(axis=1), axis=0)\n    X = pd.merge(X, c, on='bidder_id', how='left')\n    \n\n    #b = bids[['bidder_id', 'country']].groupby(['bidder_id', 'country']).country.nunique().reset_index()\n    #b = bids[['bidder_id', 'country']].groupby(['bidder_id', 'country']).size().reset_index()\n    \n\n    \n#     a = bids.groupby(['bidder_id', 'country']).size().reset_index()\n#     a= a.rename(columns = {0:'c_count'})\n#     a.groupby(['bidder_id']).sort('c_count').agg(lambda x: x.iloc[0])\n#     \n#     a = bids.groupby(['bidder_id']).country.apply(mode())    \n#     a= a.rename(columns = {0:'bids_in_country'})\n#     a.groupby(['bidder_id']).agg(lambda x:x.value_counts())\n    \n    #bids[bids.bidder_id=='0053b78cde37c4384a20d2da9aa4272aym4pb'][bids.auction=='1l3p3'][['time', 'country']]\n    \n#     hum = np.histogram(X[X.outcome==0].countries_per_bidder_per_auction_max.values, bins=10, normed=True, range=[0,10])\n#     bot = np.histogram(X[X.outcome==1].countries_per_bidder_per_auction_max.values, bins=10, normed=True, range=[0,10])\n#     plt.plot(hum[1][0:-1], hum[0], 'g.-')\n#     plt.plot(hum[1][0:-1], bot[0], 'b.-')\n\n    return X\n\n\n    \ndef log_entropy(x):\n    e = np.sum(np.log(np.array(range(1,np.sum(x)))))\n    for i in x:\n        e -= np.sum(np.log(np.array(range(1,i))))\n    \n    return e    \n\n\n#X = urls(X, bids)\ndef urls(X, bids):\n    '''\n    number of unique referring urls used by bidder\n    unique referring urls / total_bids\n    '''\n    b = bids.groupby('bidder_id').url.nunique().reset_index().fillna(1)\n    b= b.rename(columns = {'url':'n_urls'})\n    X = pd.merge(X, b, on='bidder_id', how='left')\n    b = bids.groupby(['bidder_id']).url.count().reset_index().fillna(1)\n    b= b.rename(columns = {'url':'n_bids_url'})\n    X = pd.merge(X, b, on='bidder_id', how='left')\n    X['f_urls'] = X['n_urls']/X['n_bids_url']\n\n    nbids =bids.groupby(['bidder_id']).size().reset_index()\n    nbids= nbids.rename(columns = {0:'nbids'})\n             \n    a =bids.groupby(['bidder_id', 'url']).size().reset_index()\n    a= a.rename(columns = {0:'url_counts'})\n    a['urls_used'] = 1.0\n    \n    \n    a = pd.merge(a, nbids, on='bidder_id', how='left')\n    a['f_urls'] = a['url_counts']/a['nbids']\n    \n    b = a.groupby('url').urls_used.count().reset_index()\n    print 'num urls ', b[b.urls_used > 18].url.values.shape[0]\n     \n    repeat_urls = b[b.urls_used > 18].url.values\n     \n    c = a[a.url.isin(repeat_urls)].pivot('bidder_id','url','f_urls').reset_index()\n    X = pd.merge(X, c, on='bidder_id', how='left')\n    X.iloc[:, -len(repeat_urls):] = X.iloc[:, -len(repeat_urls):].fillna(0)\n         \n    b = bids.groupby(['bidder_id', 'url']).size().reset_index()\n    b= b.rename(columns = {0:'url_count'})\n    c = b.groupby('bidder_id').url_count.apply(log_entropy).reset_index()\n    c= c.rename(columns = {'url_count':'url_entropy'})\n    \n    X = pd.merge(X, c, on='bidder_id', how='left')    \n    \n    X['f_urls'].fillna(1.0)\n        \n    return X\n    \n    \ndef merch(X, bids):\n    '''\n    type of merchandise on first page visited by bidder\n    '''\n    a = bids.groupby('bidder_id').first().reset_index()\n    b = pd.get_dummies(a['merchandise'], columns='merchandise')\n    X = pd.concat([X, b], axis=1)\n    \n    return X\n        \n\n    \ndef ip(X,bids):\n    bots = list(X[X.outcome==1].bidder_id.values)\n    humans = list(X[X.outcome==0].bidder_id.values)\n\n\n    \n    '''\n    IP fingerprint\n    uses a clustering algorithm to group bidders which use a similar set of IPs\n    it's quite time consuming\n    '''\n    '''\n    nbids = bids.groupby('bidder_id').size().reset_index()\n    nbids= nbids.rename(columns = {0:'n_bids'})\n    \n    b = bids[bids.bidder_id.isin(bots+humans)].groupby('ip').bidder_id.nunique().reset_index()\n    ip_many_users = b[b.bidder_id >= 2].ip.values\n    c = bids[bids.bidder_id.isin(bots+humans)].loc[bids.ip.isin(ip_many_users), :].groupby(['bidder_id', 'ip']).size().reset_index()\n    #c['v'] = 1\n    c= c.rename(columns = {0:'counts'})\n    c = pd.merge(c, nbids, on='bidder_id', how='left')\n    c['f'] = c['counts']/c['n_bids']\n    d = c.pivot('bidder_id', 'ip', 'f').reset_index().fillna(0)\n    print 'starting clustering'\n    brc = MiniBatchKMeans(n_clusters=100, init='k-means++', max_iter=100, batch_size=100, verbose=0, compute_labels=True, random_state=None, tol=0.0, max_no_improvement=10, init_size=None, n_init=3, reassignment_ratio=0.01)\n    brc.fit(d.values[:, 1:])\n\n    c = bids.loc[bids.ip.isin(ip_many_users), :].groupby(['bidder_id', 'ip']).size().reset_index()\n    c['v'] = 1\n    d = c.pivot('bidder_id', 'ip', 'v').reset_index().fillna(0)    \n    clusters = brc.predict(d.values[:, 1:])\n    print 'done clustering'\n    \n    e = d[['bidder_id']]\n    e['ip_cluster'] = np.nan\n    e.loc[:,['ip_cluster']] = pd.Series(clusters, index=e.index)\n\n    \n    X= pd.merge(X, e[['bidder_id', 'ip_cluster']], on='bidder_id', how='left')\n    X['ip_cluster'].fillna(-1, inplace=True)\n    y = pd.get_dummies(X['ip_cluster'])\n    print 'ip groupings size ', y.shape[1]\n    X = X.join(y)\n    X = X.drop('ip_cluster', 1)\n\n    print 'done ip fingerprint'\n    '''\n    \n    \n\n    '''\n    ips per auction for each bidder\n    '''\n    a = bids[['bidder_id', 'auction', 'ip']].groupby(['bidder_id','auction', 'ip']).size().reset_index()\n    a= a.rename(columns = {0:'bids_per_auction_per_ip'})\n    b = a.groupby(['bidder_id', 'auction']).bids_per_auction_per_ip.apply(log_entropy).reset_index()\n    b= b.rename(columns = {'bids_per_auction_per_ip':'bids_per_auction_per_ip_entropy'})    \n    c = b.groupby('bidder_id').bids_per_auction_per_ip_entropy.median().reset_index()\n    c= c.rename(columns = {'bids_per_auction_per_ip_entropy':'bids_per_auction_per_ip_entropy_median'})\n    X = pd.merge(X, c[['bidder_id','bids_per_auction_per_ip_entropy_median' ]], on='bidder_id', how='left')    \n    c = b.groupby('bidder_id').bids_per_auction_per_ip_entropy.mean().reset_index()\n    c= c.rename(columns = {'bids_per_auction_per_ip_entropy':'bids_per_auction_per_ip_entropy_mean'})\n    X = pd.merge(X, c[['bidder_id','bids_per_auction_per_ip_entropy_mean' ]], on='bidder_id', how='left')    \n        \n    b = bids[['bidder_id', 'auction', 'ip']].groupby(['bidder_id','auction']).ip.nunique().reset_index()\n    b= b.rename(columns = {'ip':'ips_per_bidder_per_auction'})    \n#     b = pd.merge(b,a, on=['bidder_id', 'auction'], how='left')\n#     b['f_ips_per_bidder_per_auction'] = b['ips_per_bidder_per_auction']/b['bids_per_auction']\n    \n    c = b.groupby('bidder_id').ips_per_bidder_per_auction.median().reset_index()\n    c= c.rename(columns = {'ips_per_bidder_per_auction':'ips_per_bidder_per_auction_median'})\n    X = pd.merge(X, c[['bidder_id', 'ips_per_bidder_per_auction_median']], on='bidder_id', how='left')\n    c = b.groupby('bidder_id').ips_per_bidder_per_auction.mean().reset_index()\n    c= c.rename(columns = {'ips_per_bidder_per_auction':'ips_per_bidder_per_auction_mean'})\n    X = pd.merge(X, c[['bidder_id', 'ips_per_bidder_per_auction_mean']], on='bidder_id', how='left')\n\n    '''\n    fraction of bids placed by user from IP which is rare\n    '''\n    #b = bids.groupby(['ip', 'bidder_id']).size().reset_index()\n    b = bids.groupby('ip').bidder_id.nunique().reset_index()\n    ip_many_users = b[b.bidder_id > 1].ip.values\n    #ip_some_users = b[b.bidder_id < 300].ip.values\n    #ip_few_users = b[b.bidder_id <= 50].ip.values\n    ip_only_one_user = b[b.bidder_id == 1].ip.values\n    \n    bids['ip2'] = np.nan\n    bids.loc[bids.ip.isin(ip_many_users), ['ip2']] = 'many'\n    #bids.loc[bids.ip.isin(ip_some_users), ['ip2']] = 'some'\n    #bids.loc[bids.ip.isin(ip_few_users), ['ip2']] = 'few'\n    bids.loc[bids.ip.isin(ip_only_one_user), ['ip2']] = 'only_one_user'\n    \n    c = bids.groupby(['bidder_id','ip2']).size().reset_index()\n    c = c.rename(columns = {0: 'counts'})\n    d = c.pivot('bidder_id', 'ip2', 'counts').reset_index().fillna(0)\n    d['ip_only_one_user_counts'] = 1*d['only_one_user']\n    d.iloc[:,1:] = d.iloc[:,1:].div(d.iloc[:,1:].sum(axis=1), axis=0)\n    X = pd.merge(X, d, on='bidder_id', how='left')\n    X.iloc[:, -(d.shape[1]-1):].fillna(0, inplace=True)\n    X = X.drop('many', 1)\n    #X = X.drop('some', 1)\n    #X = X.drop('few', 1)\n    print 'finished building ip pivot table', d.shape[1]\n    \n    \n    '''\n    does the bidder use a network that has a bot on it? (where the bot is not that bidder)\n    what fraction of networks used by the bidder have bots on them? (where the bot is not that bidder)\n    '''\n    \n    bids = pd.merge(bids, X[['bidder_id', 'outcome']], on='bidder_id', how='left')\n    bids['outcome'] = 1.0*(bids['outcome'] == 1)\n    b = bids.groupby(['ip', 'bidder_id']).outcome.mean().reset_index()\n    b.outcome = 1.0*(b.outcome)\n    c = b.groupby('ip').outcome.sum().reset_index()        \n    c['nbots_on_ip'] = c.outcome\n    bids = pd.merge(bids, c[['ip', 'nbots_on_ip']], on='ip', how='left')\n    bids['on_ip_that_has_a_bot'] = 1*((bids['nbots_on_ip'] - bids['outcome']) >= 1)\n    bids['on_ip_that_has_3bots'] = 1*((bids['nbots_on_ip'] - bids['outcome']) >= 3)\n    bids = bids.drop('outcome', 1)\n    bids = bids.drop('nbots_on_ip', 1)\n    \n    b = bids.groupby(['bidder_id', 'ip']).on_ip_that_has_a_bot.mean().reset_index()\n    c = b.groupby('bidder_id').on_ip_that_has_a_bot.mean().reset_index()\n    X = pd.merge(X, c[['bidder_id', 'on_ip_that_has_a_bot']], on='bidder_id', how='left')\n    X['on_ip_that_has_a_bot_mean'] = X['on_ip_that_has_a_bot'].fillna(0)\n    X['on_ip_that_has_a_bot'] = 1*(X['on_ip_that_has_a_bot'].fillna(0) > 0)\n    \n#      b = bids.groupby(['bidder_id', 'ip']).on_ip_that_has_3bots.mean().reset_index()\n#      c = b.groupby('bidder_id').on_ip_that_has_3bots.mean().reset_index()    \n#      X = pd.merge(X, c[['bidder_id', 'on_ip_that_has_3bots']], on='bidder_id', how='left')\n#     X['on_ip_that_has_3bots_mean'] = X['on_ip_that_has_3bots'].fillna(0)\n#     X['on_ip_that_has_3bots'] = 1*(X['on_ip_that_has_3bots'].fillna(0) > 0)\n    \n    '''\n    IP entropy\n    '''\n    b = bids.groupby(['bidder_id', 'ip']).size().reset_index()\n    b= b.rename(columns = {0:'ip_count'})\n    c = b.groupby('bidder_id').ip_count.apply(log_entropy).reset_index()\n    c= c.rename(columns = {'ip_count':'ip_entropy'})\n    X = pd.merge(X, c, on='bidder_id', how='left')\n    \n\n\n    '''\n    time between subsequent bids on different IPs for each user\n    '''\n    for week in [0, 2, 4]:\n        b = bids[bids.week==week].sort(['bidder_id', 'time'])[['bidder_id', 'time', 'ip']]\n        b['bidder_id_prev'] = pd.Series(np.append([np.nan], b.bidder_id.values[0:-1]), index=b.index)\n        b['ip_prev'] = pd.Series(np.append([np.nan], b.ip.values[0:-1]), index=b.index)\n        dt = np.append([np.nan], b.time.values[0:-1])\n        b['time_prev'] = pd.Series(dt, index=b.index)\n        b['dt'] = b['time'] - b['time_prev']\n                \n        if week==0:\n            c = [b[b.bidder_id == b.bidder_id_prev][b.ip != b.ip_prev]]\n        else:\n            c.append(b[b.bidder_id == b.bidder_id_prev][b.ip != b.ip_prev])\n\n    c = pd.concat(c)    \n    d = c.groupby('bidder_id').dt.median().reset_index()\n    d = d.rename(columns = {'dt': 'dt_change_ip_median'})\n    X = pd.merge(X, d[['bidder_id', 'dt_change_ip_median']], on='bidder_id', how='left')\n    \n    \n    '''\n    time between bids on the same IP for each user\n    '''\n    for week in [0, 2, 4]:\n        b = bids[bids.week==week].sort(['bidder_id', 'time'])[['bidder_id', 'time', 'ip']]\n        b['bidder_id_prev'] = pd.Series(np.append([np.nan], b.bidder_id.values[0:-1]), index=b.index)\n        b['ip_prev'] = pd.Series(np.append([np.nan], b.ip.values[0:-1]), index=b.index)\n        dt = np.append([np.nan], b.time.values[0:-1])\n        b['time_prev'] = pd.Series(dt, index=b.index)\n        b['dt'] = b['time'] - b['time_prev']\n                \n        if week==0:\n            c = [b[b.bidder_id == b.bidder_id_prev][b.ip == b.ip_prev]]\n        else:\n            c.append(b[b.bidder_id == b.bidder_id_prev][b.ip == b.ip_prev])\n\n    c = pd.concat(c)    \n    d = c.groupby('bidder_id').dt.median().reset_index()\n    d = d.rename(columns = {'dt': 'dt_same_ip_median'})\n    X = pd.merge(X, d[['bidder_id', 'dt_same_ip_median']], on='bidder_id', how='left')\n    \n\n\n    \n    #X['dt_change_ip-dt_same_ip'] = X['dt_change_ip_median'] - X['dt_same_ip_median']\n    #X.loc[:,'dt_change_ip_median'] = X.loc[:,'dt_change_ip_median'].fillna(30.0*one_day)\n    #X.loc[:,'dt_same_ip_median'] = X.loc[:,'dt_same_ip_median'].fillna(30.0*one_day)\n    #X.loc[:,'dt_change_ip-dt_same_ip'] = X.loc[:,'dt_change_ip-dt_same_ip'].fillna(30.0*one_day)\n    \n    X['dt_change_ip_median'].fillna(-1, inplace=True)\n    X['dt_same_ip_median'].fillna(-1, inplace=True)\n#     X['dt_change_ip-dt_same_ip'].fillna(-31*one_day, inplace=True)\n    \n    \n    '''\n    giant pivot table of overlapping IPs\n    '''\n    '''\n    q = bids[bids.bidder_id.isin(bots+humans)].groupby('ip').bidder_id.nunique().reset_index()\n    ip_many_users = q[q.bidder_id >= 3].ip.values\n    \n    b = bids.groupby(['ip','bidder_id',]).size().reset_index()\n    b = b[b.ip.isin(ip_many_users)]\n    b = b.rename(columns = {0:'count'})\n    b.drop('count', 1, inplace=True)\n\n    b2 = bids[bids.bidder_id.isin(bots+humans)].groupby(['ip','bidder_id',]).size().reset_index()\n    b2 = b2[b2.ip.isin(ip_many_users)]\n    b2 = b2.rename(columns = {0:'count'})\n    b2.drop('count', 1, inplace=True)    \n    \n    b2 = b2.rename(columns = {'bidder_id':'bidder_id2'})\n    b2['v'] = 1\n    c = pd.merge(b,b2, on='ip', how='left')\n    d = c.pivot_table(index='bidder_id', columns='bidder_id2', values='v')\n    d = d.reset_index().fillna(0)\n    X = pd.merge(X, d, on='bidder_id', how='left')\n    X.iloc[:, -d.shape[1]-1:].fillna(0, inplace=True)\n    print 'finished cluster fingerprint'\n    '''\n\n    \n    return X\n    \n    \n    \n    \n#X = build_X(bids, bot_or_human)     \ndef build_X(bids, bot_or_human):\n    X = bot_or_human\n    X = X.drop('payment_account', 1)\n    X = X.drop('address', 1)\n\n    bids['day'] = (np.floor((bids['time'] - startt)/one_day))\n    bids['week'] = (np.floor(bids['day']/7.0))\n        \n    #print bids.bidder_id[0]\n    print 'starting ips'\n    X = ip(X, bids)        \n    print 'starting bid order'\n    X = bid_order(X, bids)\n    print 'starting dt'\n    X = dt(X, bids)                \n    print 'startin day'\n    X = day(X, bids)\n    print 'starting n_bids'\n    X = n_bids(X, bids)\n    print 'starting urls'\n    X = urls(X, bids)\n    #print 'starting bid order'\n    #X = bid_order(X, bids)\n    print 'starting countries'\n    X = user_countries_per_auction(X, bids)\n    print 'starting merch'\n    X = merch(X, bids)        \n    \n\n    return X\n\n\n\n\ndef mean_over_cols(x):\n    s = np.sum(x)\n    x['f_bids_first_third'] = x[0]/s\n    x['f_bids_second_third'] = x[1]/s\n    x['f_bids_third_third'] = x[2]/s\n    return x\n    \n\n\n\n\n\ndef make(X):\n    output = X[X.outcome==-1]['bidder_id'].reset_index()\n    print X.shape\n    X.drop(X[X.outcome >=0][X.n_bids.isnull()].index, inplace=True)\n    print X.shape\n    \n    #X = X.sort(['n_bids', 'dt_others_median'])\n    X = X.sort(['dt_others_median'])\n    #X = X.sort(['on_ip_that_has_a_bot_mean', 'dt_others_median'])\n    X = X.fillna(method='pad')\n#      X = X.sort(['on_ip_that_has_a_bot_mean'])\n#      X = X.fillna(method='pad')\n    X = X.fillna(method='backfill')\n    X.sort_index(inplace=True)\n    X = X.fillna(0)\n    \n    X = X.drop('most_common_country', 1)\n    X = X.drop('bidder_id', 1)\n        \n    #clf0 = GradientBoostingClassifier(n_estimators=100, learning_rate=0.01, max_depth=4, min_samples_split=2)\n    #clf0 = RandomForestClassifier(n_estimators=100, max_depth=4, min_samples_split=2, random_state=0) \n    \n    #clf = SGDClassifier(loss=\"modified_huber\", penalty=\"elasticnet\", n_iter=20000, class_weight='auto', alpha=0.1, epsilon=0.01)\n    #clf = SVC(C=1.0, class_weight='auto', probability=True, kernel='rbf', gamma=0.0, tol=1e-1)\n    #clf = AdaBoostClassifier(n_estimators=3000)\n    #clf = GradientBoostingClassifier(n_estimators=3000, learning_rate=0.01, max_depth=None, min_samples_leaf=1)\n    #clf = ExtraTreesClassifier(n_estimators=3000, max_depth=None, min_samples_leaf=1, random_state=0)\n    \n    #clf = RandomForestClassifier(n_estimators=2500, max_depth=None, min_samples_leaf=1, random_state=0, criterion='entropy') \n    clf = []\n    nclf = 5\n    for i in xrange(nclf):\n        clf.append(RandomForestClassifier(n_estimators=800, max_depth=None, min_samples_leaf=1, random_state=i, criterion='entropy') )\n\n    scores = []\n    scores0 = []\n    \n    Yb = X['outcome'].values\n    Xb = 1.0*X.drop('outcome', 1)\n    \n    columns = Xb.columns\n    Xb = preprocessing.normalize(Xb.values, axis=0)\n    first_test_item = X[X.outcome>=0].shape[0]\n    \n    \n    X_test = Xb[first_test_item:, :]\n    print 'X_test shape ', X_test.shape\n    Y = Yb[0:first_test_item]\n    X = Xb[0:first_test_item, :]\n    \n    \n    n = int(0.8*X.shape[0])\n    all = range(X.shape[0])\n                    \n    for i in range(100):\n        \n        np.random.shuffle(all)\n        train = all[0:n]\n        valid = all[n:]\n\n        X_train = Xb[train, :]\n        X_valid = Xb[valid, :]\n        Y_train = Yb[train]\n        Y_valid = Yb[valid]\n        \n\n\n        X_train2 = X_train\n        X_valid2 = X_valid\n          \n        print '--- ', i\n        pred_valid = np.zeros(X_valid2.shape[0])\n        \n        for j in range(nclf):\n            clf[j].fit(X_train2, Y_train) #, sample_weight=(0.95*Y_train+0.05)\n            #pred_train = clf.predict_proba(X_train2) \n            #print pred_train[[21,50,51, 100, 101, 102, 103]]\n            #print Y_train.shape, pred_train.shape\n            #print roc_auc_score(Y_train, pred_train[:,1])\n            a = clf[j].predict_proba(X_valid2)[:,1]\n            pred_valid += a\n            print roc_auc_score(Y_valid, a)\n        pred_valid = 1.0*pred_valid/nclf    \n        \n        s = roc_auc_score(Y_valid, pred_valid)\n        print s\n        scores.append(s)\n        \n\n        \n    print np.mean(scores), 2.0*np.std(scores)    \n    #print np.max(clf.feature_importances_), np.argmax(clf.feature_importances_)\n    #for i in range(20):\n    #    print columns[i], clf.feature_importances_[i]\n    \n    v = np.argsort(clf[0].feature_importances_)[::-1]\n    for i in range(40):\n        print columns[v[i]], clf[0].feature_importances_[v[i]]\n\n    if len(scores) == 100:\n        print np.sort(np.array(scores))[10]\n        print np.sort(np.array(scores))[25]\n        print np.sort(np.array(scores))[50]\n\n    \n    print 'making test predictions'\n    \n    X_train = X\n    Y_train = Y\n\n    \n    pred_test = np.zeros(X_test.shape[0])\n    for j in range(nclf):\n        clf[j].fit(X_train, Y_train) #, sample_weight=(0.95*Y_train+0.05)\n        a = clf[j].predict_proba(X_test)[:,1]\n        pred_test += a\n        \n    pred_test = 1.0*pred_test/nclf\n            \n\n    \n    #output probability that bidder is a robot\n    \n    print pred_test.shape, \n    output['prediction'] = pd.Series(pred_test, index=output.index)\n    output.drop('index', 1)\n    \n    output.to_csv('submission.csv', sep=',', index=False, header=True, columns=['bidder_id', 'prediction'])\n    \n    \n    \n\n    return scores\n\n\n\n\n\n### This part is my own(Chan) code\nbids, bot_or_human = load()\nX = build_X(bids, bot_or_human)\nmake(X)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T17:11:39.330027Z","iopub.execute_input":"2025-07-22T17:11:39.330490Z","iopub.status.idle":"2025-07-22T18:04:17.451138Z","shell.execute_reply.started":"2025-07-22T17:11:39.330457Z","shell.execute_reply":"2025-07-22T18:04:17.449095Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\nsource activate py27\n\npython\nimport pickle\nimport pandas as pd\n\n#chan    \nwith open('bidsX.pkl', 'rb') as f: # 'wb' for write binary mode\n    bids, X = pickle.load(f)\n    \na = bids.groupby(['bidder_id']).country.value_counts().reset_index()\nprint(a.head(20))\nb = a.groupby(['bidder_id']).agg(lambda x: x.iloc[0]).reset_index()\nprint(b.head(20))\nb= b.rename(columns = {'country':'most_common_country'})\nprint(b.head(20))\nX = pd.merge(X, b[['bidder_id','most_common_country']], on='bidder_id', how='left')    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T18:15:07.502931Z","iopub.execute_input":"2025-07-22T18:15:07.503564Z","iopub.status.idle":"2025-07-22T18:17:28.430700Z","shell.execute_reply.started":"2025-07-22T18:15:07.503507Z","shell.execute_reply":"2025-07-22T18:17:28.429658Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}