{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-25T10:08:20.129452Z","iopub.execute_input":"2021-07-25T10:08:20.130168Z","iopub.status.idle":"2021-07-25T10:08:20.148087Z","shell.execute_reply.started":"2021-07-25T10:08:20.130050Z","shell.execute_reply":"2021-07-25T10:08:20.146592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport gzip\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:08:20.150116Z","iopub.execute_input":"2021-07-25T10:08:20.150853Z","iopub.status.idle":"2021-07-25T10:08:21.225855Z","shell.execute_reply.started":"2021-07-25T10:08:20.150801Z","shell.execute_reply":"2021-07-25T10:08:21.224716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = 40428967  #total number of records in the clickstream data \nsample_size = 3000000\nskip_values = sorted(random.sample(range(1,n), n-sample_size))\n\nwith gzip.open('../input/avazu-ctr-prediction/train.gz') as f:\n    train = pd.read_csv(f,skiprows = skip_values)\ntrain['hour'] = pd.to_datetime(train['hour'],format = '%y%m%d%H')","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:08:21.228393Z","iopub.execute_input":"2021-07-25T10:08:21.228823Z","iopub.status.idle":"2021-07-25T10:11:41.033360Z","shell.execute_reply.started":"2021-07-25T10:08:21.228776Z","shell.execute_reply":"2021-07-25T10:11:41.032519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:11:41.035065Z","iopub.execute_input":"2021-07-25T10:11:41.035655Z","iopub.status.idle":"2021-07-25T10:11:41.071499Z","shell.execute_reply.started":"2021-07-25T10:11:41.035611Z","shell.execute_reply":"2021-07-25T10:11:41.070481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:11:41.072933Z","iopub.execute_input":"2021-07-25T10:11:41.073541Z","iopub.status.idle":"2021-07-25T10:11:41.094135Z","shell.execute_reply.started":"2021-07-25T10:11:41.073495Z","shell.execute_reply":"2021-07-25T10:11:41.092948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:11:41.095957Z","iopub.execute_input":"2021-07-25T10:11:41.096858Z","iopub.status.idle":"2021-07-25T10:11:43.794390Z","shell.execute_reply.started":"2021-07-25T10:11:41.096757Z","shell.execute_reply":"2021-07-25T10:11:43.793455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe(include = 'all')","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:11:43.795584Z","iopub.execute_input":"2021-07-25T10:11:43.795870Z","iopub.status.idle":"2021-07-25T10:11:56.369510Z","shell.execute_reply.started":"2021-07-25T10:11:43.795843Z","shell.execute_reply":"2021-07-25T10:11:56.368348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop('click',axis=1)\ny = train.click","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:11:56.372043Z","iopub.execute_input":"2021-07-25T10:11:56.372383Z","iopub.status.idle":"2021-07-25T10:11:56.749276Z","shell.execute_reply.started":"2021-07-25T10:11:56.372353Z","shell.execute_reply":"2021-07-25T10:11:56.748157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.3,random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:11:56.751139Z","iopub.execute_input":"2021-07-25T10:11:56.751480Z","iopub.status.idle":"2021-07-25T10:12:01.779583Z","shell.execute_reply.started":"2021-07-25T10:11:56.751450Z","shell.execute_reply":"2021-07-25T10:12:01.778240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = X.select_dtypes(include = ['int','float']).columns.tolist()\ncategorical_cols = X.select_dtypes(include = ['object']).columns.tolist()\nprint(num_cols)\nprint(categorical_cols)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:12:01.781085Z","iopub.execute_input":"2021-07-25T10:12:01.781468Z","iopub.status.idle":"2021-07-25T10:12:02.195157Z","shell.execute_reply.started":"2021-07-25T10:12:01.781433Z","shell.execute_reply":"2021-07-25T10:12:02.194081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hashing\n","metadata":{}},{"cell_type":"code","source":"for col in categorical_cols:\n\tX_train[col] = X_train[col].apply(lambda x: hash(x))\n    \nfor col in categorical_cols:\n    X_test[col] = X_test[col].apply(lambda x:hash(x))","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:12:02.198246Z","iopub.execute_input":"2021-07-25T10:12:02.198545Z","iopub.status.idle":"2021-07-25T10:12:28.271579Z","shell.execute_reply.started":"2021-07-25T10:12:02.198516Z","shell.execute_reply":"2021-07-25T10:12:28.270739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scaling the Numerical Values","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nstd = StandardScaler()\nX_train[num_cols] = std.fit_transform(X_train[num_cols])\nX_test[num_cols] = std.transform(X_test[num_cols])","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:12:28.272849Z","iopub.execute_input":"2021-07-25T10:12:28.273373Z","iopub.status.idle":"2021-07-25T10:12:42.292563Z","shell.execute_reply.started":"2021-07-25T10:12:28.273340Z","shell.execute_reply":"2021-07-25T10:12:42.291655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train['user_info'] = X_train.device_ip + X_train.device_model + X_train.device_id\nX_train = X_train.drop(['device_id','device_ip','device_model','id','hour'],axis=1)\n    \nX_train['device_info'] = X_train.device_type + X_train.banner_pos + X_train.device_conn_type\nX_train = X_train.drop(['banner_pos','device_conn_type','device_type'],axis=1)\n\nX_test['user_info'] = X_test.device_ip + X_test.device_model + X_test.device_id\nX_test = X_test.drop(['device_id','device_ip','device_model','id','hour'],axis=1)\n    \nX_test['device_info'] = X_test.device_type + X_test.banner_pos + X_test.device_conn_type\nX_test = X_test.drop(['banner_pos','device_conn_type','device_type'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:12:42.293817Z","iopub.execute_input":"2021-07-25T10:12:42.294384Z","iopub.status.idle":"2021-07-25T10:12:43.128909Z","shell.execute_reply.started":"2021-07-25T10:12:42.294347Z","shell.execute_reply":"2021-07-25T10:12:43.127787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model - Decision Tree Classifier","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\ntree = DecisionTreeClassifier(max_depth = 10)\ntree.fit(X_train,y_train)\nprint('Train Score:',tree.score(X_train,y_train))\nprint('Test Score:',tree.score(X_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:12:43.130480Z","iopub.execute_input":"2021-07-25T10:12:43.130887Z","iopub.status.idle":"2021-07-25T10:12:59.628406Z","shell.execute_reply.started":"2021-07-25T10:12:43.130844Z","shell.execute_reply":"2021-07-25T10:12:59.627361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve,confusion_matrix,precision_score,recall_score,roc_auc_score\ny_score = tree.predict_proba(X_test)\nfpr, tpr, thresholds = roc_curve(y_test, y_score[:, 1])\nroc_auc_score = roc_auc_score(y_test,y_score[:,1])\nprint(roc_auc_score)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:12:59.630030Z","iopub.execute_input":"2021-07-25T10:12:59.630358Z","iopub.status.idle":"2021-07-25T10:13:00.262004Z","shell.execute_reply.started":"2021-07-25T10:12:59.630327Z","shell.execute_reply":"2021-07-25T10:13:00.261064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = tree.predict(X_test)\nprecision = precision_score(y_test, y_pred, average = 'weighted')\nrecall = recall_score(y_test, y_pred, average = 'weighted')\nprint(\"Precision: %s, Recall: %s\" %(precision, recall))","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:13:00.263415Z","iopub.execute_input":"2021-07-25T10:13:00.263729Z","iopub.status.idle":"2021-07-25T10:13:01.215245Z","shell.execute_reply.started":"2021-07-25T10:13:00.263697Z","shell.execute_reply":"2021-07-25T10:13:01.214081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matrix = confusion_matrix(y_test,y_pred)\ntn, fp, fn, tp = matrix.ravel()\nprint(matrix)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:13:01.216691Z","iopub.execute_input":"2021-07-25T10:13:01.217090Z","iopub.status.idle":"2021-07-25T10:13:02.488704Z","shell.execute_reply.started":"2021-07-25T10:13:01.217048Z","shell.execute_reply":"2021-07-25T10:13:02.487929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for max_depth_val in [2, 3, 5, 10, 15, 20]:\n    clf = DecisionTreeClassifier(max_depth = max_depth_val)\n    print(\"Evaluating tree with max_depth = %s\" %(max_depth_val))\n    y_pred = tree.fit(X_train,y_train).predict(X_test) \n    print(\"Confusion matrix: \")\n    print(confusion_matrix(y_test, y_pred))\n    prec = precision_score(y_test, y_pred, average = 'weighted')\n    recall = recall_score(y_test, y_pred, average = 'weighted')\n    print(\"Precision: %s, Recall: %s\" %(prec, recall))","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:13:02.489820Z","iopub.execute_input":"2021-07-25T10:13:02.490330Z","iopub.status.idle":"2021-07-25T10:14:47.725767Z","shell.execute_reply.started":"2021-07-25T10:13:02.490297Z","shell.execute_reply":"2021-07-25T10:14:47.724463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold,cross_val_score\nfor max_depth_val in [3, 5, 10]:\n    k_fold = KFold(n_splits = 4)\n    clf = DecisionTreeClassifier(max_depth = max_depth_val)\n    print(\"Evaluating Decision Tree for max_depth = %s\" %(max_depth_val))\n    y_pred = tree.fit(X_train, y_train).predict(X_test) \n  \n    cv_precision = cross_val_score(clf, X_train, y_train, cv = k_fold, scoring = 'recall_weighted')\n    precision = recall_score(y_test, y_pred, average = 'weighted')\n    print(\"Cross validation Recall: %s\" %(cv_precision.mean()))\n    print(\"Test Recall: %s\" %(precision.mean()))","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:14:47.727362Z","iopub.execute_input":"2021-07-25T10:14:47.727772Z","iopub.status.idle":"2021-07-25T10:17:11.824119Z","shell.execute_reply.started":"2021-07-25T10:14:47.727728Z","shell.execute_reply":"2021-07-25T10:17:11.822748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import RandomizedSearchCV\n\nparam_grid = {\n    'bootstrap': [True],\n    'max_depth': [10], \n    'n_estimators':[2,5,10,20,50],\n    'min_samples_split': [2, 3, 4],\n    'max_features' : ['log2']\n}\n\nrf = RandomForestClassifier(random_state=42)\n\nrf_search = RandomizedSearchCV(estimator = rf, param_distributions=param_grid, \n                          cv = 5, n_jobs = -1, verbose = 2, n_iter = 10)\n\nmodel = rf_search.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:17:11.826162Z","iopub.execute_input":"2021-07-25T10:17:11.826513Z","iopub.status.idle":"2021-07-25T10:33:42.708620Z","shell.execute_reply.started":"2021-07-25T10:17:11.826479Z","shell.execute_reply":"2021-07-25T10:33:42.707617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:33:42.710235Z","iopub.execute_input":"2021-07-25T10:33:42.710701Z","iopub.status.idle":"2021-07-25T10:33:42.719100Z","shell.execute_reply.started":"2021-07-25T10:33:42.710667Z","shell.execute_reply":"2021-07-25T10:33:42.718253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_rf = RandomForestClassifier(n_estimators= 20,\n min_samples_split= 2,\n max_features= 'log2',\n max_depth= 10,\n bootstrap= True,\n random_state=42)\nclf_rf.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:33:42.721455Z","iopub.execute_input":"2021-07-25T10:33:42.721892Z","iopub.status.idle":"2021-07-25T10:35:02.512699Z","shell.execute_reply.started":"2021-07-25T10:33:42.721845Z","shell.execute_reply":"2021-07-25T10:35:02.511738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_pred = clf_rf.predict(X_test)\n\nprint(confusion_matrix(y_test, y_test_pred))\nprec = precision_score(y_test, y_test_pred, average = 'weighted')\nrecall = recall_score(y_test, y_test_pred, average = 'weighted')\nprint(\"Precision: %s, Recall: %s\" %(prec, recall))","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:35:02.516580Z","iopub.execute_input":"2021-07-25T10:35:02.516904Z","iopub.status.idle":"2021-07-25T10:35:06.659583Z","shell.execute_reply.started":"2021-07-25T10:35:02.516874Z","shell.execute_reply":"2021-07-25T10:35:06.658386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_prob = clf_rf.predict_proba(X_test)\nrf_fpr,rf_tpr,_ = roc_curve(y_test,rf_prob[:,1])","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:35:06.661417Z","iopub.execute_input":"2021-07-25T10:35:06.661850Z","iopub.status.idle":"2021-07-25T10:35:08.934186Z","shell.execute_reply.started":"2021-07-25T10:35:06.661806Z","shell.execute_reply":"2021-07-25T10:35:08.933098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import roc_curve,auc,confusion_matrix,precision_score,recall_score,roc_auc_score\nparams = {\n    \"objective\": \"binary:logistic\",\n    \"booster\" : \"gbtree\",\n    \"eval_metric\": \"logloss\",\n    \"eta\":0.1,\n    \"max_depth\": 8,\n    \"subsample\": 0.8,\n    \"colsample_bytree\": 0.8,\n    \"silent\": 1,\n}\nxgclf=xgb.XGBClassifier(**params)\nxgclf.fit(X_train, y_train,\n        eval_set=[(X_test, y_test)],\n        eval_metric='logloss',\n        verbose=False)\nxgpred=xgclf.predict_proba(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:35:08.935546Z","iopub.execute_input":"2021-07-25T10:35:08.935875Z","iopub.status.idle":"2021-07-25T10:39:25.691702Z","shell.execute_reply.started":"2021-07-25T10:35:08.935841Z","shell.execute_reply":"2021-07-25T10:39:25.690722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics \n#y_score = xgclf.predict_proba(x_test)\n#fpr, tpr, thresholds = roc_curve(y_test, y_score[:, 1])\nroc_auc_score = metrics.roc_auc_score(y_test,xgpred[:,1])\nprint(roc_auc_score)\nprint(xgclf.score(X_test,y_test))\nxgb_fpr,xgb_tpr,_ = roc_curve(y_test,xgpred[:,1])","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:39:25.693201Z","iopub.execute_input":"2021-07-25T10:39:25.693788Z","iopub.status.idle":"2021-07-25T10:39:27.697152Z","shell.execute_reply.started":"2021-07-25T10:39:25.693750Z","shell.execute_reply":"2021-07-25T10:39:27.696181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold,cross_val_score\n# Set up k-fold\nk_fold = KFold(n_splits = 5)\n\n# Evaluate precision and recall for each fold\nprecision = cross_val_score(\n  xgclf, X_train, y_train, cv = k_fold, scoring = 'precision_weighted')\nrecall = cross_val_score(\n  xgclf, X_train, y_train, cv = k_fold, scoring = 'recall_weighted')\nprint(\"Precision scores: %s\" %(precision.mean())) \nprint(\"Recall scores: %s\" %(recall.mean()))\nprint(k_fold)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T10:39:27.698461Z","iopub.execute_input":"2021-07-25T10:39:27.698776Z","iopub.status.idle":"2021-07-25T11:10:57.033339Z","shell.execute_reply.started":"2021-07-25T10:39:27.698745Z","shell.execute_reply":"2021-07-25T11:10:57.032062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_probs = [0 for i in range(len(y_test))]\np_fpr, p_tpr, _ = roc_curve(y_test, random_probs, pos_label=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T11:10:57.035184Z","iopub.execute_input":"2021-07-25T11:10:57.035710Z","iopub.status.idle":"2021-07-25T11:10:57.315576Z","shell.execute_reply.started":"2021-07-25T11:10:57.035658Z","shell.execute_reply":"2021-07-25T11:10:57.314687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('seaborn')\nplt.plot(fpr,tpr,linestyle = '--',color = 'green',label='Decision Tree')\nplt.plot(rf_fpr,rf_tpr,linestyle = '--',color = 'yellow',label='Random Forest')\nplt.plot(xgb_fpr,xgb_tpr,linestyle = '--',color = 'orange',label='XGBoost')\nplt.plot(p_fpr,p_tpr,linestyle='--',color = 'blue')\nplt.legend()\nplt.xlabel('FPR')\nplt.ylabel('TPR')\nplt.title('AUC-ROC Curve')","metadata":{"execution":{"iopub.status.busy":"2021-07-25T11:10:57.316939Z","iopub.execute_input":"2021-07-25T11:10:57.317606Z","iopub.status.idle":"2021-07-25T11:10:57.770405Z","shell.execute_reply.started":"2021-07-25T11:10:57.317557Z","shell.execute_reply":"2021-07-25T11:10:57.769079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}