{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-31T17:26:46.072171Z","iopub.execute_input":"2023-05-31T17:26:46.073055Z","iopub.status.idle":"2023-05-31T17:26:46.115111Z","shell.execute_reply.started":"2023-05-31T17:26:46.073007Z","shell.execute_reply":"2023-05-31T17:26:46.114285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nnum_records = 40428967\nsample_size = 20000000\nskip_values = sorted(random.sample(range(1,num_records), num_records - sample_size))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:26:46.116749Z","iopub.execute_input":"2023-05-31T17:26:46.117311Z","iopub.status.idle":"2023-05-31T17:27:39.047711Z","shell.execute_reply.started":"2023-05-31T17:26:46.117274Z","shell.execute_reply":"2023-05-31T17:27:39.0466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clicks = pd.read_csv(\"../input/avazu-ctr-prediction/train.gz\", skiprows=skip_values)\nclicks.head(2)\nsample_size = 0","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:27:39.049029Z","iopub.execute_input":"2023-05-31T17:27:39.049405Z","iopub.status.idle":"2023-05-31T17:30:34.064447Z","shell.execute_reply.started":"2023-05-31T17:27:39.049375Z","shell.execute_reply":"2023-05-31T17:30:34.060031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = list(clicks.columns)\na, b = cols.index('click'), cols.index('id')\ncols[b], cols[a] = cols[a], cols[b]\nclicks = clicks[cols]\nclicks.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:30:34.071002Z","iopub.execute_input":"2023-05-31T17:30:34.07192Z","iopub.status.idle":"2023-05-31T17:30:44.420036Z","shell.execute_reply.started":"2023-05-31T17:30:34.071835Z","shell.execute_reply":"2023-05-31T17:30:44.416915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cl1 = clicks[clicks['click'] == 1]\ncl2 = clicks[clicks['click'] == 0].sample(n=len(cl1))\nclicks = pd.concat([cl1, cl2])","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:30:44.424326Z","iopub.execute_input":"2023-05-31T17:30:44.426356Z","iopub.status.idle":"2023-05-31T17:31:01.557682Z","shell.execute_reply.started":"2023-05-31T17:30:44.426237Z","shell.execute_reply":"2023-05-31T17:31:01.556561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clicks['id'] = clicks['id'].astype('uint64')\nD = 2 ** 10\ni = 'click'\nclicks[i] = clicks[i].astype('uint8')\nclicks[clicks < 0 ] = 0\nfor i in clicks.columns[5:16]:\n    print(i)\n    clicks[i] = clicks[i].apply(lambda x: hash(f'{i}={x}') % D)\nfor i in clicks.columns[2:]:\n    clicks[i] = clicks[i].astype('uint32')","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:31:01.559258Z","iopub.execute_input":"2023-05-31T17:31:01.559602Z","iopub.status.idle":"2023-05-31T17:32:42.943878Z","shell.execute_reply.started":"2023-05-31T17:31:01.559574Z","shell.execute_reply":"2023-05-31T17:32:42.942697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clicks.info()\n","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:42.945311Z","iopub.execute_input":"2023-05-31T17:32:42.946987Z","iopub.status.idle":"2023-05-31T17:32:43.367674Z","shell.execute_reply.started":"2023-05-31T17:32:42.946957Z","shell.execute_reply":"2023-05-31T17:32:43.366511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = clicks","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:43.36893Z","iopub.execute_input":"2023-05-31T17:32:43.369741Z","iopub.status.idle":"2023-05-31T17:32:43.381959Z","shell.execute_reply.started":"2023-05-31T17:32:43.369704Z","shell.execute_reply":"2023-05-31T17:32:43.381135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dict_data = {}\n# l = len(clicks)\n# cl = clicks[clicks['click'] == 1]\n# for i in clicks.columns[2:]:\n#     t = {}\n#     for j in clicks[i].unique():\n#         t[j] = [len(cl[cl[i] == j]), len(clicks[clicks[i] == j])]\n#     dict_data[i] = t\n","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:43.383329Z","iopub.execute_input":"2023-05-31T17:32:43.384094Z","iopub.status.idle":"2023-05-31T17:32:43.393323Z","shell.execute_reply.started":"2023-05-31T17:32:43.384058Z","shell.execute_reply":"2023-05-31T17:32:43.39245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pickle\n# with open(f\"dict_data\", \"wb\") as fp:   #Pickling\n#     pickle.dump(dict_data, fp)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T17:32:43.397733Z","iopub.execute_input":"2023-05-31T17:32:43.398372Z","iopub.status.idle":"2023-05-31T17:32:43.408989Z","shell.execute_reply.started":"2023-05-31T17:32:43.398345Z","shell.execute_reply":"2023-05-31T17:32:43.408255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:43.410406Z","iopub.execute_input":"2023-05-31T17:32:43.411002Z","iopub.status.idle":"2023-05-31T17:32:46.735282Z","shell.execute_reply.started":"2023-05-31T17:32:43.410974Z","shell.execute_reply":"2023-05-31T17:32:46.734479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = clicks.sample(n=4000000)\n# test = clicks.sample(n=1000000)\nresult[result < 0 ] = 0","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:46.736703Z","iopub.execute_input":"2023-05-31T17:32:46.737264Z","iopub.status.idle":"2023-05-31T17:32:47.1628Z","shell.execute_reply.started":"2023-05-31T17:32:46.737235Z","shell.execute_reply":"2023-05-31T17:32:47.161621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split \nx = result.iloc[:, 2:]\ny = result.iloc[:, 0]\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:47.16416Z","iopub.execute_input":"2023-05-31T17:32:47.164532Z","iopub.status.idle":"2023-05-31T17:32:55.531143Z","shell.execute_reply.started":"2023-05-31T17:32:47.164506Z","shell.execute_reply":"2023-05-31T17:32:55.530176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:55.532544Z","iopub.execute_input":"2023-05-31T17:32:55.532867Z","iopub.status.idle":"2023-05-31T17:32:55.903799Z","shell.execute_reply.started":"2023-05-31T17:32:55.532839Z","shell.execute_reply":"2023-05-31T17:32:55.902714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.naive_bayes import ComplementNB \n\nmodel = ComplementNB( force_alpha=True).fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:55.905505Z","iopub.execute_input":"2023-05-31T17:32:55.906196Z","iopub.status.idle":"2023-05-31T17:32:59.769826Z","shell.execute_reply.started":"2023-05-31T17:32:55.906159Z","shell.execute_reply":"2023-05-31T17:32:59.768623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test = x_test.where(x_test < 0, 0)\n\npredicted = model.predict(x_test) \n \nprint(np.mean(predicted == y_test)) ","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:32:59.771128Z","iopub.execute_input":"2023-05-31T17:32:59.771497Z","iopub.status.idle":"2023-05-31T17:33:00.06129Z","shell.execute_reply.started":"2023-05-31T17:32:59.771468Z","shell.execute_reply":"2023-05-31T17:33:00.059827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" from sklearn.metrics import confusion_matrix \n \n print(confusion_matrix(y_test, predicted) ) ","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:00.063392Z","iopub.execute_input":"2023-05-31T17:33:00.064484Z","iopub.status.idle":"2023-05-31T17:33:00.288255Z","shell.execute_reply.started":"2023-05-31T17:33:00.064428Z","shell.execute_reply":"2023-05-31T17:33:00.287155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predicted = model.predict(x_train) \n \n# print(np.mean(predicted == y_train)) \n# print(confusion_matrix(y_train, predicted) ) ","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:00.289699Z","iopub.execute_input":"2023-05-31T17:33:00.290349Z","iopub.status.idle":"2023-05-31T17:33:00.294546Z","shell.execute_reply.started":"2023-05-31T17:33:00.290313Z","shell.execute_reply":"2023-05-31T17:33:00.293546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cols = train.columns[2:]\n# cl = train[train['click'] == 1]\n# test['ver'] = [0 for i in range(len(test))]\n# z = 0\n# for unit in test.index:\n#     if z%50000==0:\n#         print(z)\n#     z += 1\n#     p = 1\n#     for i in cols:\n#         j = test[i][unit]\n#         p *= len(cl[cl[i] == j])\n#     p *= len(cl)\n#     test['ver'][unit] = p","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:00.295916Z","iopub.execute_input":"2023-05-31T17:33:00.296252Z","iopub.status.idle":"2023-05-31T17:33:00.305909Z","shell.execute_reply.started":"2023-05-31T17:33:00.296196Z","shell.execute_reply":"2023-05-31T17:33:00.305128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def logloss_crutch(y_true, y_pred, eps=1e-15):\n    return - (y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))\ndef analysis(res):\n    real_click = (res['click'].sum() / res.count())[0]\n    ver = 1 - real_click\n    true_click = (res[(res['click'] == 1) & (res['ver'] > ver)].count() / res.count())[0]\n    true_not_click = (res[(res['click'] == 0) & (res['ver'] < ver)].count() / res.count())[0]\n    pred_click = true_click + true_not_click\n    print(f'real: {real_click}')\n    print(f'ver: {ver}')\n    print(f'true_click: {true_click}')\n    print(f'true_not_click: {true_not_click}')\n    print(f'---')\n    print(f'pred_click: {pred_click }')\n    print('Logloss: %f' % logloss_crutch(real_click, true_click))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:00.306916Z","iopub.execute_input":"2023-05-31T17:33:00.307816Z","iopub.status.idle":"2023-05-31T17:33:00.318253Z","shell.execute_reply.started":"2023-05-31T17:33:00.307783Z","shell.execute_reply":"2023-05-31T17:33:00.31735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(predicted)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:00.319528Z","iopub.execute_input":"2023-05-31T17:33:00.319867Z","iopub.status.idle":"2023-05-31T17:33:00.784446Z","shell.execute_reply.started":"2023-05-31T17:33:00.319841Z","shell.execute_reply":"2023-05-31T17:33:00.783485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {'click': y_test, 'ver': predicted}\nframe2 = pd.DataFrame(data, columns=['click', 'ver'])","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:00.785953Z","iopub.execute_input":"2023-05-31T17:33:00.786362Z","iopub.status.idle":"2023-05-31T17:33:00.834995Z","shell.execute_reply.started":"2023-05-31T17:33:00.786326Z","shell.execute_reply":"2023-05-31T17:33:00.834079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(frame2['ver'])","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:00.837138Z","iopub.execute_input":"2023-05-31T17:33:00.838053Z","iopub.status.idle":"2023-05-31T17:33:01.01829Z","shell.execute_reply.started":"2023-05-31T17:33:00.838023Z","shell.execute_reply":"2023-05-31T17:33:01.017343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame2['ver'][4303510] = 1","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:01.019741Z","iopub.execute_input":"2023-05-31T17:33:01.020783Z","iopub.status.idle":"2023-05-31T17:33:01.107856Z","shell.execute_reply.started":"2023-05-31T17:33:01.020754Z","shell.execute_reply":"2023-05-31T17:33:01.106762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame2","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:01.109467Z","iopub.execute_input":"2023-05-31T17:33:01.110354Z","iopub.status.idle":"2023-05-31T17:33:01.122329Z","shell.execute_reply.started":"2023-05-31T17:33:01.110314Z","shell.execute_reply":"2023-05-31T17:33:01.121064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"analysis(frame2)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:01.123557Z","iopub.execute_input":"2023-05-31T17:33:01.123834Z","iopub.status.idle":"2023-05-31T17:33:01.173943Z","shell.execute_reply.started":"2023-05-31T17:33:01.12381Z","shell.execute_reply":"2023-05-31T17:33:01.1729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn import metrics\n\nlogreg = LogisticRegression()\nlogreg.fit(x_train, y_train)\ny_pred = logreg.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:01.175222Z","iopub.execute_input":"2023-05-31T17:33:01.175635Z","iopub.status.idle":"2023-05-31T17:33:21.208069Z","shell.execute_reply.started":"2023-05-31T17:33:01.175607Z","shell.execute_reply":"2023-05-31T17:33:21.206526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame2['ver'] = y_pred","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:21.223935Z","iopub.execute_input":"2023-05-31T17:33:21.225077Z","iopub.status.idle":"2023-05-31T17:33:21.235691Z","shell.execute_reply.started":"2023-05-31T17:33:21.225012Z","shell.execute_reply":"2023-05-31T17:33:21.23424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frame2","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:21.237756Z","iopub.execute_input":"2023-05-31T17:33:21.238258Z","iopub.status.idle":"2023-05-31T17:33:21.26791Z","shell.execute_reply.started":"2023-05-31T17:33:21.2382Z","shell.execute_reply":"2023-05-31T17:33:21.26632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"analysis(frame2)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:21.269809Z","iopub.execute_input":"2023-05-31T17:33:21.271921Z","iopub.status.idle":"2023-05-31T17:33:21.332259Z","shell.execute_reply.started":"2023-05-31T17:33:21.271867Z","shell.execute_reply":"2023-05-31T17:33:21.331032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" print(confusion_matrix(y_test, frame2['ver']) ) ","metadata":{"execution":{"iopub.status.busy":"2023-05-31T17:33:21.333675Z","iopub.execute_input":"2023-05-31T17:33:21.334097Z","iopub.status.idle":"2023-05-31T17:33:21.544879Z","shell.execute_reply.started":"2023-05-31T17:33:21.334047Z","shell.execute_reply":"2023-05-31T17:33:21.543657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}