{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/avazu-ctr-prediction/train.gz',header=0, nrows=300000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train.columns:\n    print(col,'unique values',np.sort(train[col].unique()),'\\n')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop(columns=['id','click','id', 'hour', 'device_id', 'device_ip']).values\nY = train['click'].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_train = int(300000 * 0.9)\nX_train = X[:n_train]\nY_train = Y[:n_train]\nX_test = X[n_train:]\nY_test = Y[n_train:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\nonehotenc = OneHotEncoder(handle_unknown='ignore')\nX_train_enc = onehotenc.fit_transform(X_train)\nX_test_enc = onehotenc.transform(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\ndecision_tree = DecisionTreeClassifier(criterion='gini', min_samples_split=30)\ndecision_tree.fit(X_train_enc, Y_train)\npos_prob = decision_tree.predict_proba(X_test_enc)[:, 1]\nprint(pos_prob)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nprint(f'The ROC AUC on testing set is: {roc_auc_score(Y_test, pos_prob):.3f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import tree\nimport matplotlib.pyplot as plt\nplt.figure(figsize=(12,12))\n_ = tree.plot_tree(decision_tree,fontsize=10,max_depth=4) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters = {'max_depth': [3, 10, None]}\nfrom sklearn.model_selection import GridSearchCV\ngrid_search = GridSearchCV(decision_tree, parameters, n_jobs=-1, cv=3, scoring='roc_auc')\ngrid_search.fit(X_train_enc, Y_train)\nprint(grid_search.best_params_)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decision_tree_best = grid_search.best_estimator_\npos_prob = decision_tree_best.predict_proba(X_test_enc)[:, 1]\n\nfrom sklearn.metrics import roc_auc_score\nprint(f'The ROC AUC on testing set is: {roc_auc_score(Y_test, pos_prob):.3f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\nrandom_forest = RandomForestClassifier(n_estimators=100, criterion='gini', min_samples_split=30, n_jobs=-1)\ngrid_search = GridSearchCV(random_forest, parameters, n_jobs=-1, cv=3, scoring='roc_auc')\ngrid_search.fit(X_train_enc, Y_train)\nprint(grid_search.best_params_)\nprint(grid_search.best_score_)\n\nrandom_forest_best = grid_search.best_estimator_\npos_prob = random_forest_best.predict_proba(X_test_enc)[:, 1]\nprint(f'The ROC AUC on testing set is: {roc_auc_score(Y_test, pos_prob):.3f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nxgbst = xgb.XGBClassifier(learning_rate=0.1, max_depth=10, n_estimators=1000,use_label_encoder=False)\n\nxgbst.fit(X_train_enc, Y_train)\npos_prob = xgbst.predict_proba(X_test_enc)[:, 1]\n\nprint(f'The ROC AUC on testing set is: {roc_auc_score(Y_test, pos_prob):.3f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_test1 = {\n 'max_depth':range(3,10,2),\n 'min_child_weight':range(1,6,2)\n}\ngsearch1 = GridSearchCV(estimator = xgbst, \n param_grid = param_test1, scoring='roc_auc',n_jobs=-1, cv=5)\ngsearch1.fit(X_train_enc, Y_train)\ngsearch1.best_params_, gsearch1.best_score_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgbst = xgb.XGBClassifier(learning_rate=0.1, max_depth=9, n_estimators=1000,use_label_encoder=False)\nxgbst.fit(X_train_enc, Y_train)\npos_prob = xgbst.predict_proba(X_test_enc)[:, 1]\n\nprint(f'The ROC AUC on testing set is: {roc_auc_score(Y_test, pos_prob):.3f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}