{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"dockerImageVersionId":30120,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-04T12:21:16.014248Z","iopub.execute_input":"2022-09-04T12:21:16.014598Z","iopub.status.idle":"2022-09-04T12:21:16.02384Z","shell.execute_reply.started":"2022-09-04T12:21:16.014568Z","shell.execute_reply":"2022-09-04T12:21:16.022813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport gzip\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:21:16.025365Z","iopub.execute_input":"2022-09-04T12:21:16.025927Z","iopub.status.idle":"2022-09-04T12:21:16.169341Z","shell.execute_reply.started":"2022-09-04T12:21:16.025894Z","shell.execute_reply":"2022-09-04T12:21:16.168295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = 40428967  #total number of records in the clickstream data \nsample_size = 3000000\nskip_values = sorted(random.sample(range(1,n), n-sample_size))\n\nwith gzip.open('../input/avazu-ctr-prediction/train.gz') as f:\n    train = pd.read_csv(f,skiprows = skip_values)\ntrain['hour'] = pd.to_datetime(train['hour'],format = '%y%m%d%H')","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:21:16.17155Z","iopub.execute_input":"2022-09-04T12:21:16.171956Z","iopub.status.idle":"2022-09-04T12:24:03.97176Z","shell.execute_reply.started":"2022-09-04T12:21:16.171915Z","shell.execute_reply":"2022-09-04T12:24:03.970704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:03.973597Z","iopub.execute_input":"2022-09-04T12:24:03.974083Z","iopub.status.idle":"2022-09-04T12:24:04.006078Z","shell.execute_reply.started":"2022-09-04T12:24:03.974049Z","shell.execute_reply":"2022-09-04T12:24:04.005113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:04.00748Z","iopub.execute_input":"2022-09-04T12:24:04.007859Z","iopub.status.idle":"2022-09-04T12:24:04.026833Z","shell.execute_reply.started":"2022-09-04T12:24:04.007805Z","shell.execute_reply":"2022-09-04T12:24:04.025675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:04.028636Z","iopub.execute_input":"2022-09-04T12:24:04.029098Z","iopub.status.idle":"2022-09-04T12:24:05.146224Z","shell.execute_reply.started":"2022-09-04T12:24:04.029046Z","shell.execute_reply":"2022-09-04T12:24:05.145258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe(include = 'all')","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:05.148451Z","iopub.execute_input":"2022-09-04T12:24:05.148758Z","iopub.status.idle":"2022-09-04T12:24:12.168397Z","shell.execute_reply.started":"2022-09-04T12:24:05.148727Z","shell.execute_reply":"2022-09-04T12:24:12.167454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop('click',axis=1)\ny = train.click","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:12.170232Z","iopub.execute_input":"2022-09-04T12:24:12.170548Z","iopub.status.idle":"2022-09-04T12:24:12.631939Z","shell.execute_reply.started":"2022-09-04T12:24:12.170516Z","shell.execute_reply":"2022-09-04T12:24:12.630997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.3,random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:12.633335Z","iopub.execute_input":"2022-09-04T12:24:12.633624Z","iopub.status.idle":"2022-09-04T12:24:16.849125Z","shell.execute_reply.started":"2022-09-04T12:24:12.633598Z","shell.execute_reply":"2022-09-04T12:24:16.848292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = X.select_dtypes(include = ['int','float']).columns.tolist()\ncategorical_cols = X.select_dtypes(include = ['object']).columns.tolist()\nprint(num_cols)\nprint(categorical_cols)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:16.850268Z","iopub.execute_input":"2022-09-04T12:24:16.850543Z","iopub.status.idle":"2022-09-04T12:24:17.381017Z","shell.execute_reply.started":"2022-09-04T12:24:16.850516Z","shell.execute_reply":"2022-09-04T12:24:17.380089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hashing\n","metadata":{}},{"cell_type":"code","source":"for col in categorical_cols:\n\tX_train[col] = X_train[col].apply(lambda x: hash(x))\n    \nfor col in categorical_cols:\n    X_test[col] = X_test[col].apply(lambda x:hash(x))","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:17.382166Z","iopub.execute_input":"2022-09-04T12:24:17.382483Z","iopub.status.idle":"2022-09-04T12:24:36.54449Z","shell.execute_reply.started":"2022-09-04T12:24:17.382453Z","shell.execute_reply":"2022-09-04T12:24:36.54353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scaling the Numerical Values","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nstd = StandardScaler()\nX_train[num_cols] = std.fit_transform(X_train[num_cols])\nX_test[num_cols] = std.transform(X_test[num_cols])","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:36.545649Z","iopub.execute_input":"2022-09-04T12:24:36.545914Z","iopub.status.idle":"2022-09-04T12:24:47.063115Z","shell.execute_reply.started":"2022-09-04T12:24:36.545888Z","shell.execute_reply":"2022-09-04T12:24:47.062094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train['user_info'] = X_train.device_ip + X_train.device_model + X_train.device_id\nX_train = X_train.drop(['device_id','device_ip','device_model','id','hour'],axis=1)\n    \nX_train['device_info'] = X_train.device_type + X_train.banner_pos + X_train.device_conn_type\nX_train = X_train.drop(['banner_pos','device_conn_type','device_type'],axis=1)\n\nX_test['user_info'] = X_test.device_ip + X_test.device_model + X_test.device_id\nX_test = X_test.drop(['device_id','device_ip','device_model','id','hour'],axis=1)\n    \nX_test['device_info'] = X_test.device_type + X_test.banner_pos + X_test.device_conn_type\nX_test = X_test.drop(['banner_pos','device_conn_type','device_type'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:47.066263Z","iopub.execute_input":"2022-09-04T12:24:47.066572Z","iopub.status.idle":"2022-09-04T12:24:47.898638Z","shell.execute_reply.started":"2022-09-04T12:24:47.066543Z","shell.execute_reply":"2022-09-04T12:24:47.89785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model - Decision Tree Classifier","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\ntree = DecisionTreeClassifier(max_depth = 10)\ntree.fit(X_train,y_train)\nprint('Train Score:',tree.score(X_train,y_train))\nprint('Test Score:',tree.score(X_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:24:47.899932Z","iopub.execute_input":"2022-09-04T12:24:47.900414Z","iopub.status.idle":"2022-09-04T12:25:16.472898Z","shell.execute_reply.started":"2022-09-04T12:24:47.900369Z","shell.execute_reply":"2022-09-04T12:25:16.471728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve,confusion_matrix,precision_score,recall_score,roc_auc_score\ny_score = tree.predict_proba(X_test)\nfpr, tpr, thresholds = roc_curve(y_test, y_score[:, 1])\nroc_auc_score = roc_auc_score(y_test,y_score[:,1])\nprint(roc_auc_score)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:25:16.474213Z","iopub.execute_input":"2022-09-04T12:25:16.47462Z","iopub.status.idle":"2022-09-04T12:25:17.09173Z","shell.execute_reply.started":"2022-09-04T12:25:16.474583Z","shell.execute_reply":"2022-09-04T12:25:17.090544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = tree.predict(X_test)\nprecision = precision_score(y_test, y_pred, average = 'weighted')\nrecall = recall_score(y_test, y_pred, average = 'weighted')\nprint(\"Precision: %s, Recall: %s\" %(precision, recall))","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:25:17.092951Z","iopub.execute_input":"2022-09-04T12:25:17.093296Z","iopub.status.idle":"2022-09-04T12:25:18.03628Z","shell.execute_reply.started":"2022-09-04T12:25:17.093255Z","shell.execute_reply":"2022-09-04T12:25:18.035114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matrix = confusion_matrix(y_test,y_pred)\ntn, fp, fn, tp = matrix.ravel()\nprint(matrix)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:25:18.037682Z","iopub.execute_input":"2022-09-04T12:25:18.03799Z","iopub.status.idle":"2022-09-04T12:25:18.815085Z","shell.execute_reply.started":"2022-09-04T12:25:18.037928Z","shell.execute_reply":"2022-09-04T12:25:18.813997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for max_depth_val in [2, 3, 5, 10, 15, 20]:\n    clf = DecisionTreeClassifier(max_depth = max_depth_val)\n    print(\"Evaluating tree with max_depth = %s\" %(max_depth_val))\n    y_pred = tree.fit(X_train,y_train).predict(X_test) \n    print(\"Confusion matrix: \")\n    print(confusion_matrix(y_test, y_pred))\n    prec = precision_score(y_test, y_pred, average = 'weighted')\n    recall = recall_score(y_test, y_pred, average = 'weighted')\n    print(\"Precision: %s, Recall: %s\" %(prec, recall))","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:25:18.816336Z","iopub.execute_input":"2022-09-04T12:25:18.816612Z","iopub.status.idle":"2022-09-04T12:28:14.983387Z","shell.execute_reply.started":"2022-09-04T12:25:18.816584Z","shell.execute_reply":"2022-09-04T12:28:14.982044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold,cross_val_score\nfor max_depth_val in [3, 5, 10]:\n    k_fold = KFold(n_splits = 4)\n    clf = DecisionTreeClassifier(max_depth = max_depth_val)\n    print(\"Evaluating Decision Tree for max_depth = %s\" %(max_depth_val))\n    y_pred = tree.fit(X_train, y_train).predict(X_test) \n  \n    cv_precision = cross_val_score(clf, X_train, y_train, cv = k_fold, scoring = 'recall_weighted')\n    precision = recall_score(y_test, y_pred, average = 'weighted')\n    print(\"Cross validation Recall: %s\" %(cv_precision.mean()))\n    print(\"Test Recall: %s\" %(precision.mean()))","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:28:14.987426Z","iopub.execute_input":"2022-09-04T12:28:14.987779Z","iopub.status.idle":"2022-09-04T12:32:25.743756Z","shell.execute_reply.started":"2022-09-04T12:28:14.987737Z","shell.execute_reply":"2022-09-04T12:32:25.74259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nfrom sklearn import datasets\nfrom sklearn.tree import DecisionTreeClassifier \nfrom sklearn import tree","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:32:25.745287Z","iopub.execute_input":"2022-09-04T12:32:25.74559Z","iopub.status.idle":"2022-09-04T12:32:25.750356Z","shell.execute_reply.started":"2022-09-04T12:32:25.745561Z","shell.execute_reply":"2022-09-04T12:32:25.749421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iris = datasets.load_iris()\nX = iris.data\ny = iris.target\nclf = DecisionTreeClassifier(random_state=1234)\nmodel = clf.fit(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:32:25.751637Z","iopub.execute_input":"2022-09-04T12:32:25.751926Z","iopub.status.idle":"2022-09-04T12:32:26.113115Z","shell.execute_reply.started":"2022-09-04T12:32:25.751887Z","shell.execute_reply":"2022-09-04T12:32:26.112262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_representation = tree.export_text(clf)\nprint(text_representation)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:32:26.114309Z","iopub.execute_input":"2022-09-04T12:32:26.114616Z","iopub.status.idle":"2022-09-04T12:32:26.120827Z","shell.execute_reply.started":"2022-09-04T12:32:26.114583Z","shell.execute_reply":"2022-09-04T12:32:26.11996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/avazu-ctr-prediction\", \"w\") as fout:\n    fout.write(text_representation)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:32:26.121786Z","iopub.execute_input":"2022-09-04T12:32:26.122068Z","iopub.status.idle":"2022-09-04T12:32:26.170999Z","shell.execute_reply.started":"2022-09-04T12:32:26.122041Z","shell.execute_reply":"2022-09-04T12:32:26.169842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(25,20))\n_ = tree.plot_tree(clf, \n                   feature_names=iris.feature_names,  \n                   class_names=iris.target_names,\n                   filled=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:32:44.181341Z","iopub.execute_input":"2022-09-04T12:32:44.181698Z","iopub.status.idle":"2022-09-04T12:32:45.535584Z","shell.execute_reply.started":"2022-09-04T12:32:44.181667Z","shell.execute_reply":"2022-09-04T12:32:45.534326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig.savefig(\"decistion_tree.png\")","metadata":{"execution":{"iopub.status.busy":"2022-09-04T12:32:50.191253Z","iopub.execute_input":"2022-09-04T12:32:50.191715Z","iopub.status.idle":"2022-09-04T12:32:50.700329Z","shell.execute_reply.started":"2022-09-04T12:32:50.191674Z","shell.execute_reply":"2022-09-04T12:32:50.699418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}