{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\nfrom sklearn.model_selection import GridSearchCV\nfrom xgboost import XGBClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport gzip","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-08T05:52:26.775472Z","iopub.execute_input":"2023-04-08T05:52:26.776379Z","iopub.status.idle":"2023-04-08T05:52:28.221335Z","shell.execute_reply.started":"2023-04-08T05:52:26.776307Z","shell.execute_reply":"2023-04-08T05:52:28.220505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n\ntest_file = '../input/avazu-ctr-prediction/test.gz'\nsamplesubmision_file = '../input/avazu-ctr-prediction/sampleSubmission.gz'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-08T05:52:28.224552Z","iopub.execute_input":"2023-04-08T05:52:28.224901Z","iopub.status.idle":"2023-04-08T05:52:28.245979Z","shell.execute_reply.started":"2023-04-08T05:52:28.224870Z","shell.execute_reply":"2023-04-08T05:52:28.245148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunksize = 10 ** 6\nnum_of_chunk = 0\ntrain = pd.DataFrame()\n    \nfor chunk in pd.read_csv('../input/avazu-ctr-train/train.csv', chunksize=chunksize):\n    num_of_chunk += 1\n    train = pd.concat([train, chunk.sample(frac=.05, replace=False, random_state=123)], axis=0)\n    print('Processing Chunk No. ' + str(num_of_chunk))     \n    \ntrain.reset_index(inplace=True)\ntrain_len = len(train)\ntrain_len","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:52:28.247911Z","iopub.execute_input":"2023-04-08T05:52:28.248278Z","iopub.status.idle":"2023-04-08T05:55:39.885935Z","shell.execute_reply.started":"2023-04-08T05:52:28.248239Z","shell.execute_reply":"2023-04-08T05:55:39.884834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.concat([train, pd.read_csv(test_file, compression='gzip')]).drop(['index', 'id'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:55:39.887407Z","iopub.execute_input":"2023-04-08T05:55:39.887779Z","iopub.status.idle":"2023-04-08T05:55:59.248251Z","shell.execute_reply.started":"2023-04-08T05:55:39.887740Z","shell.execute_reply":"2023-04-08T05:55:59.247189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_date(hour):\n    y = '20'+str(hour)[:2]\n    m = str(hour)[2:4]\n    d = str(hour)[4:6]\n    return y+'-'+m+'-'+d\n\ndf['weekday'] = pd.to_datetime(df.hour.apply(get_date)).dt.dayofweek.astype(str)\n\ndef tran_hour(x):\n    x = x % 100\n    while x in [23,0]:\n        return '23-01'\n    while x in [1,2]:\n        return '01-03'\n    while x in [3,4]:\n        return '03-05'\n    while x in [5,6]:\n        return '05-07'\n    while x in [7,8]:\n        return '07-09'\n    while x in [9,10]:\n        return '09-11'\n    while x in [11,12]:\n        return '11-13'\n    while x in [13,14]:\n        return '13-15'\n    while x in [15,16]:\n        return '15-17'\n    while x in [17,18]:\n        return '17-19'\n    while x in [19,20]:\n        return '19-21'\n    while x in [21,22]:\n        return '21-23'\n\ndf['hour'] = df.hour.apply(tran_hour)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:55:59.251615Z","iopub.execute_input":"2023-04-08T05:55:59.251897Z","iopub.status.idle":"2023-04-08T05:56:17.170957Z","shell.execute_reply.started":"2023-04-08T05:55:59.251869Z","shell.execute_reply":"2023-04-08T05:56:17.170013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:56:17.173354Z","iopub.execute_input":"2023-04-08T05:56:17.173634Z","iopub.status.idle":"2023-04-08T05:56:20.288840Z","shell.execute_reply.started":"2023-04-08T05:56:17.173607Z","shell.execute_reply":"2023-04-08T05:56:20.287862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_of_feature_count = []\nfor i in df.columns[2:23].tolist():\n    print(i, ':', len(df[i].astype(str).value_counts()))\n    len_of_feature_count.append(len(df[i].astype(str).value_counts()))","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:56:20.290329Z","iopub.execute_input":"2023-04-08T05:56:20.290717Z","iopub.status.idle":"2023-04-08T05:58:44.121593Z","shell.execute_reply.started":"2023-04-08T05:56:20.290687Z","shell.execute_reply":"2023-04-08T05:58:44.120675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"need_tran_feature = df.columns[2:4].tolist() + df.columns[13:23].tolist()\n\nfor i in need_tran_feature:\n    df[i] = df[i].astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:58:44.123117Z","iopub.execute_input":"2023-04-08T05:58:44.123474Z","iopub.status.idle":"2023-04-08T05:59:35.399445Z","shell.execute_reply.started":"2023-04-08T05:58:44.123437Z","shell.execute_reply":"2023-04-08T05:59:35.398360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"obj_features = []\n\nfor i in range(len(len_of_feature_count)):\n    if len_of_feature_count[i] > 10:\n        obj_features.append(df.columns[2:23].tolist()[i])\nobj_features","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:59:35.400906Z","iopub.execute_input":"2023-04-08T05:59:35.401271Z","iopub.status.idle":"2023-04-08T05:59:35.410262Z","shell.execute_reply.started":"2023-04-08T05:59:35.401226Z","shell.execute_reply":"2023-04-08T05:59:35.409326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_describe = df.describe()\ndf_describe","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:59:35.411717Z","iopub.execute_input":"2023-04-08T05:59:35.412298Z","iopub.status.idle":"2023-04-08T05:59:42.509547Z","shell.execute_reply.started":"2023-04-08T05:59:35.412259Z","shell.execute_reply":"2023-04-08T05:59:42.508381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def obj_clean(X):\n    \n\n    def get_click_rate(x):\n        \n        temp = train[train[X.columns[0]] == x]\n        res = round((temp.click.sum() / temp.click.count()),3)\n        return res\n\n    def get_type(V, str):\n        \n        very_high = df_describe.loc['mean','click'] + 0.04\n        higher = df_describe.loc['mean','click'] + 0.02\n        lower = df_describe.loc['mean','click'] - 0.02\n        very_low = df_describe.loc['mean','click'] - 0.04\n\n        vh_type = V[V[str] > very_high].index.tolist()\n        hr_type = V[(V[str] > higher) & (V[str] < very_high)].index.tolist()\n        vl_type = V[V[str] < very_low].index.tolist()\n        lr_type = V[(V[str] < lower) & (V[str] > very_low)].index.tolist()\n\n        return vh_type, hr_type, vl_type, lr_type\n\n    def clean_function(x):\n        \n        while x in type_[0]:\n            return 'very_high'\n        while x in type_[1]:\n            return 'higher'\n        while x in type_[2]:\n            return 'very_low'\n        while x in type_[3]:\n            return 'lower'\n        return 'mid'\n        \n    print('Run: ', X.columns[0])\n    fq = X[X.columns[0]].value_counts()\n    \n    if len(fq) > 1000:\n        fq = fq[:1000]\n\n    fq = pd.DataFrame(fq)\n    fq['new_column'] = fq.index    \n    fq['click_rate'] = fq.new_column.apply(get_click_rate)\n    type_ = get_type(fq, 'click_rate')\n    return X[X.columns[0]].apply(clean_function)\n\nfor i in obj_features:    \n    df[[i]] = obj_clean(df[[i]])\n\ndf","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:59:42.511090Z","iopub.execute_input":"2023-04-08T05:59:42.511479Z","iopub.status.idle":"2023-04-08T06:30:16.781217Z","shell.execute_reply.started":"2023-04-08T05:59:42.511445Z","shell.execute_reply":"2023-04-08T06:30:16.779891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in df.columns:\n    sns.countplot(x = i, hue = \"click\", data = df)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T06:30:16.782631Z","iopub.execute_input":"2023-04-08T06:30:16.782980Z","iopub.status.idle":"2023-04-08T06:31:07.190099Z","shell.execute_reply.started":"2023-04-08T06:30:16.782944Z","shell.execute_reply":"2023-04-08T06:31:07.189203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.drop(['device_id', 'C14', 'C17', 'C19', 'C20', 'C21'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T06:31:07.191576Z","iopub.execute_input":"2023-04-08T06:31:07.192140Z","iopub.status.idle":"2023-04-08T06:31:09.170568Z","shell.execute_reply.started":"2023-04-08T06:31:07.192083Z","shell.execute_reply":"2023-04-08T06:31:09.169653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf = pd.get_dummies(df)\ntrain = df[:train_len]\ntest = df[train_len:]","metadata":{"execution":{"iopub.status.busy":"2023-04-08T06:31:09.171979Z","iopub.execute_input":"2023-04-08T06:31:09.172340Z","iopub.status.idle":"2023-04-08T06:31:19.940640Z","shell.execute_reply.started":"2023-04-08T06:31:09.172299Z","shell.execute_reply":"2023-04-08T06:31:19.939488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df","metadata":{"execution":{"iopub.status.busy":"2023-04-08T06:31:19.941989Z","iopub.execute_input":"2023-04-08T06:31:19.942363Z","iopub.status.idle":"2023-04-08T06:31:19.952672Z","shell.execute_reply.started":"2023-04-08T06:31:19.942324Z","shell.execute_reply":"2023-04-08T06:31:19.951751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_X = train[train['click'] == 0].sample(n=len(train[train['click'] == 1]), random_state=111)\npre_X = pd.concat([pre_X, train[train['click'] == 1]]).sample(frac=1)\npre_y = pre_X[['click']]\npre_X.drop(['click'], axis=1, inplace=True)\ntest.drop(['click'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T06:31:19.954217Z","iopub.execute_input":"2023-04-08T06:31:19.954849Z","iopub.status.idle":"2023-04-08T06:31:22.092411Z","shell.execute_reply.started":"2023-04-08T06:31:19.954807Z","shell.execute_reply":"2023-04-08T06:31:22.091626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_X_train, pre_X_test, pre_y_train, pre_y_test = train_test_split(pre_X, pre_y, test_size=0.20, stratify=pre_y, random_state=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T06:31:22.094072Z","iopub.execute_input":"2023-04-08T06:31:22.094439Z","iopub.status.idle":"2023-04-08T06:31:24.916531Z","shell.execute_reply.started":"2023-04-08T06:31:22.094402Z","shell.execute_reply":"2023-04-08T06:31:24.915526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\"criterion\":[\"gini\", \"entropy\"], \"max_depth\":range(1,20)}\ngrid_search = GridSearchCV(DecisionTreeClassifier(), param_grid=params, scoring='roc_auc', cv=100, verbose=1, n_jobs=-1)\ngrid_search.fit(pre_X_train, pre_y_train)\ngrid_search.best_score_, grid_search.best_estimator_, grid_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-04-08T06:31:24.917962Z","iopub.execute_input":"2023-04-08T06:31:24.918340Z","iopub.status.idle":"2023-04-08T11:11:29.485401Z","shell.execute_reply.started":"2023-04-08T06:31:24.918301Z","shell.execute_reply":"2023-04-08T11:11:29.484094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree = grid_search.best_estimator_\ntree.fit(pre_X,pre_y)\n\nfeature_importances = pd.DataFrame(tree.feature_importances_)\nfeature_importances.index = pre_X_train.columns\nfeature_importances = feature_importances.sort_values(0,ascending=False)\nfeature_importances","metadata":{"execution":{"iopub.status.busy":"2023-04-08T11:11:29.486998Z","iopub.execute_input":"2023-04-08T11:11:29.487406Z","iopub.status.idle":"2023-04-08T11:11:38.394283Z","shell.execute_reply.started":"2023-04-08T11:11:29.487371Z","shell.execute_reply":"2023-04-08T11:11:38.393226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_X_train = pre_X_train[feature_importances.index[:int(len(feature_importances)/3)]]\npre_X_test = pre_X_test[feature_importances.index[:int(len(feature_importances)/3)]]","metadata":{"execution":{"iopub.status.busy":"2023-04-08T11:11:38.395861Z","iopub.execute_input":"2023-04-08T11:11:38.396261Z","iopub.status.idle":"2023-04-08T11:11:38.420095Z","shell.execute_reply.started":"2023-04-08T11:11:38.396218Z","shell.execute_reply":"2023-04-08T11:11:38.419315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\"criterion\":[\"gini\", \"entropy\"], \"max_depth\":range(1,12)}\ngrid_search = GridSearchCV(DecisionTreeClassifier(), param_grid=params, scoring='roc_auc', cv=100, verbose=1, n_jobs=-1)\ngrid_search.fit(pre_X_train, pre_y_train)\ngrid_search.best_score_, grid_search.best_estimator_, grid_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-04-08T11:11:38.421652Z","iopub.execute_input":"2023-04-08T11:11:38.422036Z","iopub.status.idle":"2023-04-08T12:02:56.011180Z","shell.execute_reply.started":"2023-04-08T11:11:38.421996Z","shell.execute_reply":"2023-04-08T12:02:56.010232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_X = pre_X[feature_importances.index[:int(len(feature_importances)/3)]]\n\ntree = grid_search.best_estimator_\ntree.fit(pre_X,pre_y)\nfeature_importances = pd.DataFrame(tree.feature_importances_)\nfeature_importances.index = pre_X_train.columns\nfeature_importances = feature_importances.sort_values(0,ascending=False)\nfeature_importances","metadata":{"execution":{"iopub.status.busy":"2023-04-08T12:02:56.012732Z","iopub.execute_input":"2023-04-08T12:02:56.013275Z","iopub.status.idle":"2023-04-08T12:03:00.006813Z","shell.execute_reply.started":"2023-04-08T12:02:56.013234Z","shell.execute_reply":"2023-04-08T12:03:00.005882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_len = len(feature_importances[feature_importances[feature_importances.columns[0]] > 0.005])\n\ny = train[['click']]\nX = train[feature_importances[:feature_len].index]\ntest = test[feature_importances[:feature_len].index]","metadata":{"execution":{"iopub.status.busy":"2023-04-08T12:03:00.009273Z","iopub.execute_input":"2023-04-08T12:03:00.009961Z","iopub.status.idle":"2023-04-08T12:03:00.260993Z","shell.execute_reply.started":"2023-04-08T12:03:00.009915Z","shell.execute_reply":"2023-04-08T12:03:00.260208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBClassifier(tree_method = 'gpu_hist', n_jobs=-1, n_estimators=500, max_depth=11)\nmodel.fit(X,y.values.ravel())\ny_pred = model.predict(X)\nprint(\"Roc_auc_score: \",roc_auc_score(y,y_pred)*100,\"%\")\n\nconfmat = confusion_matrix(y_true=y, y_pred=y_pred, labels=[0, 1])\n\nfig, ax = plt.subplots(figsize=(2.5, 2.5))\nax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)\nfor i in range(confmat.shape[0]):\n    for j in range(confmat.shape[1]):\n        ax.text(x=j, y=i, s=confmat[i, j], va='center', ha='center')\n\nplt.xlabel('Predicted label')\nplt.ylabel('True label')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-08T12:03:00.262766Z","iopub.execute_input":"2023-04-08T12:03:00.263135Z","iopub.status.idle":"2023-04-08T12:05:39.562139Z","shell.execute_reply.started":"2023-04-08T12:03:00.263084Z","shell.execute_reply":"2023-04-08T12:05:39.561156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nsns.heatmap(confmat,annot=True,fmt='',xticklabels=[0,1],yticklabels=[0,1],linewidth=.5, cmap=sns.cubehelix_palette(as_cmap=True))\n\nplt.xlabel('Predicted label',fontsize=15)\nplt.ylabel('True label',fontsize=15)\nplt.title('Confusion Matrix',fontsize=20)\nplt.figure(figsize=(30.0,30.0),dpi=200)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T12:30:16.454966Z","iopub.execute_input":"2023-04-08T12:30:16.455381Z","iopub.status.idle":"2023-04-08T12:30:17.530733Z","shell.execute_reply.started":"2023-04-08T12:30:16.455346Z","shell.execute_reply":"2023-04-08T12:30:17.529765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.metrics import roc_curve, auc\n\n# Plot ROC curve for each model\n\nfpr3, tpr3, _ = roc_curve(y, y_pred)\nroc_auc3 = auc(fpr3, tpr3)\n\nplt.figure(figsize=(10,7))\nplt.plot(fpr3, tpr3, color='blue', lw=2, label='XGBoost (AUC = %0.2f)' % roc_auc3)\nplt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate',fontsize=15)\nplt.ylabel('True Positive Rate',fontsize=15)\nplt.title('Receiver Operating Characteristic (ROC) Curve',fontsize=20)\nplt.legend(loc=\"lower right\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-08T12:31:12.773987Z","iopub.execute_input":"2023-04-08T12:31:12.774377Z","iopub.status.idle":"2023-04-08T12:31:13.116791Z","shell.execute_reply.started":"2023-04-08T12:31:12.774341Z","shell.execute_reply":"2023-04-08T12:31:13.115819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy\nfrom sklearn import metrics","metadata":{"execution":{"iopub.status.busy":"2023-04-08T12:31:25.742962Z","iopub.execute_input":"2023-04-08T12:31:25.743342Z","iopub.status.idle":"2023-04-08T12:31:25.747623Z","shell.execute_reply.started":"2023-04-08T12:31:25.743310Z","shell.execute_reply":"2023-04-08T12:31:25.746718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import precision_recall_fscore_support\n\nprint(precision_recall_fscore_support(y, y_pred , average='macro'))\ncm=confusion_matrix(y, y_pred)\ntotal1=sum(sum(cm))\naccuracy1=(cm[0,0]+cm[1,1])/total1\nprint ('Accuracy : ', accuracy1)\nsensitivity1 = cm[0,0]/(cm[0,0]+cm[0,1])\nprint('Sensitivity : ', sensitivity1 )\nspecificity1 = cm[1,1]/(cm[1,0]+cm[1,1])\nprint('Specificity : ', specificity1)\nF1_score = metrics.f1_score(y, y_pred)\nprint('F1 Score : ', F1_score)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T12:31:33.267226Z","iopub.execute_input":"2023-04-08T12:31:33.267553Z","iopub.status.idle":"2023-04-08T12:31:36.926938Z","shell.execute_reply.started":"2023-04-08T12:31:33.267521Z","shell.execute_reply":"2023-04-08T12:31:36.925932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}