{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-20T13:13:29.385433Z","iopub.execute_input":"2021-07-20T13:13:29.385769Z","iopub.status.idle":"2021-07-20T13:13:29.395415Z","shell.execute_reply.started":"2021-07-20T13:13:29.385739Z","shell.execute_reply":"2021-07-20T13:13:29.394546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport gzip\nimport random\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:13:33.106132Z","iopub.execute_input":"2021-07-20T13:13:33.106487Z","iopub.status.idle":"2021-07-20T13:13:33.110860Z","shell.execute_reply.started":"2021-07-20T13:13:33.106453Z","shell.execute_reply":"2021-07-20T13:13:33.109682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = 40428967  #total number of records in the clickstream data \nsample_size = 1000000\nskip_values = sorted(random.sample(range(1,n), n-sample_size))\n\nparse_date = lambda val : pd.datetime.strptime(val, '%y%m%d%H')\nwith gzip.open('../input/avazu-ctr-prediction/train.gz') as f:\n    train = pd.read_csv(f,skiprows = skip_values)\ntrain['hour'] = pd.to_datetime(train['hour'],format = '%y%m%d%H')    ","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:13:35.805756Z","iopub.execute_input":"2021-07-20T13:13:35.806108Z","iopub.status.idle":"2021-07-20T13:16:14.007647Z","shell.execute_reply.started":"2021-07-20T13:13:35.806078Z","shell.execute_reply":"2021-07-20T13:16:14.006672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data=train.copy()\ny_label=train_data.pop('click')\ntrain_data=train_data.drop(['hour','id'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:16:14.009382Z","iopub.execute_input":"2021-07-20T13:16:14.009763Z","iopub.status.idle":"2021-07-20T13:16:14.325771Z","shell.execute_reply.started":"2021-07-20T13:16:14.009723Z","shell.execute_reply":"2021-07-20T13:16:14.324896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get categorical columns\ncategorical_cols = train_data.select_dtypes(\n  include = ['object']).columns.tolist()\nprint(\"Categorical columns: \")\nprint(categorical_cols)\n\n# Iterate over categorical columns and apply hash function\n","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:16:14.327503Z","iopub.execute_input":"2021-07-20T13:16:14.327813Z","iopub.status.idle":"2021-07-20T13:16:14.432816Z","shell.execute_reply.started":"2021-07-20T13:16:14.327785Z","shell.execute_reply":"2021-07-20T13:16:14.432163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in categorical_cols:\n\ttrain_data[col] = train_data[col].apply(lambda x: hash(x))","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:16:14.433792Z","iopub.execute_input":"2021-07-20T13:16:14.434193Z","iopub.status.idle":"2021-07-20T13:16:19.294979Z","shell.execute_reply.started":"2021-07-20T13:16:14.434162Z","shell.execute_reply":"2021-07-20T13:16:19.294265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:19:45.631727Z","iopub.execute_input":"2021-07-20T13:19:45.632117Z","iopub.status.idle":"2021-07-20T13:19:45.651005Z","shell.execute_reply.started":"2021-07-20T13:19:45.632079Z","shell.execute_reply":"2021-07-20T13:19:45.650318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n#X = train_data.drop(['id', 'device_id', 'device_ip', 'hour', 'click'], axis=1)\n#y = train_data.click\nx_train,x_test,y_train,y_test = train_test_split(train_data,y_label,test_size = 0.2,random_state=21)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:16:19.296087Z","iopub.execute_input":"2021-07-20T13:16:19.296636Z","iopub.status.idle":"2021-07-20T13:16:20.964768Z","shell.execute_reply.started":"2021-07-20T13:16:19.296594Z","shell.execute_reply":"2021-07-20T13:16:20.964051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nsclr = StandardScaler()\nx_train.iloc[:,2:11] = sclr.fit_transform(x_train.iloc[:,2:11])\nx_test.iloc[:,2:11] = sclr.transform(x_test.iloc[:,2:11])","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:16:20.965717Z","iopub.execute_input":"2021-07-20T13:16:20.966106Z","iopub.status.idle":"2021-07-20T13:16:23.659711Z","shell.execute_reply.started":"2021-07-20T13:16:20.966079Z","shell.execute_reply":"2021-07-20T13:16:23.658759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['user_info']=train_data.device_ip+train_data.device_model+train_data.device_id\ntrain_data=train_data.drop(['device_id','device_ip','device_model'],axis=1)\ntrain_data['devtry']=train_data.device_type+train_data.banner_pos+train_data.device_conn_type\ntrain_data=train_data.drop(['banner_pos','device_conn_type','device_type'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:16:25.962709Z","iopub.execute_input":"2021-07-20T13:16:25.963403Z","iopub.status.idle":"2021-07-20T13:16:26.328523Z","shell.execute_reply.started":"2021-07-20T13:16:25.963356Z","shell.execute_reply":"2021-07-20T13:16:26.327593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import roc_curve,auc,confusion_matrix,precision_score,recall_score,roc_auc_score\n# intialize pca and logistic regression model\npca = PCA(n_components=2)\nparams = {\n    \"objective\": \"binary:logistic\",\n    \"booster\" : \"gbtree\",\n    \"eval_metric\": \"logloss\",\n    \"eta\":0.1,\n    \"max_depth\": 8,\n    \"subsample\": 0.8,\n    \"colsample_bytree\": 0.8,\n    \"silent\": 1,\n}\nxgclf=xgb.XGBClassifier(**params)\n\n# fit and transform data\nX_train_pca = pca.fit_transform(x_train)\nX_test_pca = pca.transform(x_test)\nxgclf.fit(X_train_pca, y_train,\n        eval_set=[(X_test_pca, y_test)],\n        eval_metric='logloss',\n        verbose=False)\npca.explained_variance_ratio_\n","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:49:18.594787Z","iopub.execute_input":"2021-07-20T13:49:18.595132Z","iopub.status.idle":"2021-07-20T13:49:44.244384Z","shell.execute_reply.started":"2021-07-20T13:49:18.595103Z","shell.execute_reply":"2021-07-20T13:49:44.243510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics \nxgpred=xgclf.predict_proba(X_test_pca)\n#fpr, tpr, thresholds = roc_curve(y_test, y_score[:, 1])\nroc_auc_score = metrics.roc_auc_score(y_test,xgpred[:,1])\nprint(roc_auc_score)\nprint(xgclf.score(X_test_pca,y_test))","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:49:44.245658Z","iopub.execute_input":"2021-07-20T13:49:44.245908Z","iopub.status.idle":"2021-07-20T13:49:44.861504Z","shell.execute_reply.started":"2021-07-20T13:49:44.245882Z","shell.execute_reply":"2021-07-20T13:49:44.860450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_classif\n# determine the mutual information\nmutual_info = mutual_info_classif(train_data,y_label,discrete_features=True)\nmutual_info = pd.Series(mutual_info)\nmutual_info.index = train_data.columns\nmutual_info.sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:16:43.697730Z","iopub.execute_input":"2021-07-20T13:16:43.698082Z","iopub.status.idle":"2021-07-20T13:16:47.430295Z","shell.execute_reply.started":"2021-07-20T13:16:43.698049Z","shell.execute_reply":"2021-07-20T13:16:47.429431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import roc_curve,auc,confusion_matrix,precision_score,recall_score,roc_auc_score\nparams = {\n    \"objective\": \"binary:logistic\",\n    \"booster\" : \"gbtree\",\n    \"eval_metric\": \"logloss\",\n    \"eta\":0.1,\n    \"max_depth\": 8,\n    \"subsample\": 0.8,\n    \"colsample_bytree\": 0.8,\n    \"silent\": 1,\n}\nxgclf=xgb.XGBClassifier(**params)\nxgclf.fit(x_train, y_train,\n        eval_set=[(x_test, y_test)],\n        eval_metric='logloss',\n        verbose=False)\nxgpred=xgclf.predict_proba(x_test)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T10:13:22.884469Z","iopub.execute_input":"2021-07-20T10:13:22.884912Z","iopub.status.idle":"2021-07-20T10:14:37.940055Z","shell.execute_reply.started":"2021-07-20T10:13:22.884877Z","shell.execute_reply":"2021-07-20T10:14:37.939123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics \n#y_score = xgclf.predict_proba(x_test)\n#fpr, tpr, thresholds = roc_curve(y_test, y_score[:, 1])\nroc_auc_score = metrics.roc_auc_score(y_test,xgpred[:,1])\nprint(roc_auc_score)\nprint(xgclf.score(x_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2021-07-20T10:14:37.941684Z","iopub.execute_input":"2021-07-20T10:14:37.942236Z","iopub.status.idle":"2021-07-20T10:14:38.665633Z","shell.execute_reply.started":"2021-07-20T10:14:37.942200Z","shell.execute_reply":"2021-07-20T10:14:38.664875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold,cross_val_score\n# Set up k-fold\nk_fold = KFold(n_splits = 5)\n\n# Evaluate precision and recall for each fold\nprecision = cross_val_score(\n  xgclf, x_train, y_train, cv = k_fold, scoring = 'precision_weighted')\nrecall = cross_val_score(\n  xgclf, x_train, y_train, cv = k_fold, scoring = 'recall_weighted')\nprint(\"Precision scores: %s\" %(precision)) \nprint(\"Recall scores: %s\" %(recall))\nprint(k_fold)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T10:14:38.667200Z","iopub.execute_input":"2021-07-20T10:14:38.667574Z","iopub.status.idle":"2021-07-20T10:24:06.424167Z","shell.execute_reply.started":"2021-07-20T10:14:38.667533Z","shell.execute_reply":"2021-07-20T10:24:06.422776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:00.134671Z","iopub.execute_input":"2021-07-19T17:44:00.135172Z","iopub.status.idle":"2021-07-19T17:44:01.387416Z","shell.execute_reply.started":"2021-07-19T17:44:00.135123Z","shell.execute_reply":"2021-07-19T17:44:01.386398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis\n","metadata":{}},{"cell_type":"code","source":"print(train.columns)\nprint(train.dtypes)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:01.388723Z","iopub.execute_input":"2021-07-19T17:44:01.389039Z","iopub.status.idle":"2021-07-19T17:44:01.397382Z","shell.execute_reply.started":"2021-07-19T17:44:01.389009Z","shell.execute_reply":"2021-07-19T17:44:01.396605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## For missing values","metadata":{}},{"cell_type":"code","source":"print(train.info())\nprint(train.isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:01.398381Z","iopub.execute_input":"2021-07-19T17:44:01.398772Z","iopub.status.idle":"2021-07-19T17:44:03.239305Z","shell.execute_reply.started":"2021-07-19T17:44:01.398743Z","shell.execute_reply":"2021-07-19T17:44:03.237974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are no missing values in the dataset.","metadata":{}},{"cell_type":"markdown","source":"## id","metadata":{}},{"cell_type":"code","source":"train.drop('id',axis=1,inplace = True)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:03.242469Z","iopub.execute_input":"2021-07-19T17:44:03.242927Z","iopub.status.idle":"2021-07-19T17:44:03.537398Z","shell.execute_reply.started":"2021-07-19T17:44:03.24289Z","shell.execute_reply":"2021-07-19T17:44:03.536462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Timestamp - Hour","metadata":{}},{"cell_type":"code","source":"train['day_week'] = train['hour'].apply(lambda x: x.dayofweek)\ntrain['hour_day'] = train['hour'].apply(lambda x: x.hour)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:03.538849Z","iopub.execute_input":"2021-07-19T17:44:03.53925Z","iopub.status.idle":"2021-07-19T17:44:34.124849Z","shell.execute_reply.started":"2021-07-19T17:44:03.539214Z","shell.execute_reply":"2021-07-19T17:44:34.123543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby(['day_week','click']).size().unstack().plot(kind='bar')\nplt.title('Days of the week')\ntrain.groupby(['hour_day','click']).size().unstack().plot(kind='bar')\nplt.title('Hours of a day')","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:34.127599Z","iopub.execute_input":"2021-07-19T17:44:34.127957Z","iopub.status.idle":"2021-07-19T17:44:35.026281Z","shell.execute_reply.started":"2021-07-19T17:44:34.127924Z","shell.execute_reply":"2021-07-19T17:44:35.024647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('hour_day')['click'].sum().plot.bar(figsize=(12,6))\nplt.ylabel('Number of clicks')\nplt.title('Number of clicks by hour of day')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:35.029213Z","iopub.execute_input":"2021-07-19T17:44:35.029769Z","iopub.status.idle":"2021-07-19T17:44:35.393149Z","shell.execute_reply.started":"2021-07-19T17:44:35.029714Z","shell.execute_reply":"2021-07-19T17:44:35.391889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Click","metadata":{}},{"cell_type":"code","source":"sns.countplot(x='click',data=train)\nplt.show()\nprint(train.click.value_counts()/len(train))","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:35.394591Z","iopub.execute_input":"2021-07-19T17:44:35.394932Z","iopub.status.idle":"2021-07-19T17:44:35.718599Z","shell.execute_reply.started":"2021-07-19T17:44:35.3949Z","shell.execute_reply":"2021-07-19T17:44:35.717276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Nearly 17% is the CTR of the data provided.","metadata":{}},{"cell_type":"markdown","source":"## Banner Position","metadata":{}},{"cell_type":"code","source":"train.groupby(['banner_pos', 'click']).size().unstack().plot(kind='barh')\ntrain.groupby(['banner_pos','click']).size().unstack().iloc[2:,:].plot(kind='barh')","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:35.720129Z","iopub.execute_input":"2021-07-19T17:44:35.720543Z","iopub.status.idle":"2021-07-19T17:44:36.344306Z","shell.execute_reply.started":"2021-07-19T17:44:35.720508Z","shell.execute_reply":"2021-07-19T17:44:36.343092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"site_features = ['site_id', 'site_domain', 'site_category']\ntrain[site_features].describe()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:36.345683Z","iopub.execute_input":"2021-07-19T17:44:36.346064Z","iopub.status.idle":"2021-07-19T17:44:38.680139Z","shell.execute_reply.started":"2021-07-19T17:44:36.346032Z","shell.execute_reply":"2021-07-19T17:44:38.67905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Hashing\n","metadata":{}},{"cell_type":"code","source":"# Get categorical columns\ncategorical_cols = train.select_dtypes(\n  include = ['object']).columns.tolist()\nprint(\"Categorical columns: \")\nprint(categorical_cols)\n\n# Iterate over categorical columns and apply hash function\nfor col in categorical_cols:\n\ttrain[col] = train[col].apply(lambda x: hash(x))","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:38.681705Z","iopub.execute_input":"2021-07-19T17:44:38.682026Z","iopub.status.idle":"2021-07-19T17:44:53.813371Z","shell.execute_reply.started":"2021-07-19T17:44:38.681995Z","shell.execute_reply":"2021-07-19T17:44:53.812315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Selection","metadata":{}},{"cell_type":"code","source":"ad_corr = train.corr(method = 'pearson')\nfig,ax = plt.subplots(figsize = (20,10))\nsns.heatmap(ad_corr,\n            xticklabels=ad_corr.columns,\n            yticklabels=ad_corr.columns,\n            cmap='RdBu_r',\n            annot=True,\n            linewidth=1,ax=ax)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:44:53.814562Z","iopub.execute_input":"2021-07-19T17:44:53.814851Z","iopub.status.idle":"2021-07-19T17:45:00.925607Z","shell.execute_reply.started":"2021-07-19T17:44:53.814823Z","shell.execute_reply":"2021-07-19T17:45:00.924777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can remove the features which are irrelavant to the data, and not useful.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX = train.drop(['device_id','device_ip','hour','click'],axis=1)\ny = train.click\nx_train,x_test,y_train,y_test = train_test_split(X,y,test_size = 0.2,random_state=21)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:00.926763Z","iopub.execute_input":"2021-07-19T17:45:00.927171Z","iopub.status.idle":"2021-07-19T17:45:02.454891Z","shell.execute_reply.started":"2021-07-19T17:45:00.927139Z","shell.execute_reply":"2021-07-19T17:45:02.454022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:02.456249Z","iopub.execute_input":"2021-07-19T17:45:02.456929Z","iopub.status.idle":"2021-07-19T17:45:02.479541Z","shell.execute_reply.started":"2021-07-19T17:45:02.456881Z","shell.execute_reply":"2021-07-19T17:45:02.478441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nsclr = StandardScaler()\nx_train.iloc[:,8:15] = sclr.fit_transform(x_train.iloc[:,8:15])\nx_test.iloc[:,8:15] = sclr.transform(x_test.iloc[:,8:15])","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:02.480817Z","iopub.execute_input":"2021-07-19T17:45:02.481117Z","iopub.status.idle":"2021-07-19T17:45:08.611882Z","shell.execute_reply.started":"2021-07-19T17:45:02.481086Z","shell.execute_reply":"2021-07-19T17:45:08.610737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:08.613325Z","iopub.execute_input":"2021-07-19T17:45:08.613807Z","iopub.status.idle":"2021-07-19T17:45:08.838019Z","shell.execute_reply.started":"2021-07-19T17:45:08.61376Z","shell.execute_reply":"2021-07-19T17:45:08.837121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = DecisionTreeClassifier(max_depth=10)\nclf.fit(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:08.839052Z","iopub.execute_input":"2021-07-19T17:45:08.839484Z","iopub.status.idle":"2021-07-19T17:45:24.177792Z","shell.execute_reply.started":"2021-07-19T17:45:08.839449Z","shell.execute_reply":"2021-07-19T17:45:24.176756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train Score:',clf.score(x_train,y_train))\nprint('Test Score:',clf.score(x_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:24.179092Z","iopub.execute_input":"2021-07-19T17:45:24.179523Z","iopub.status.idle":"2021-07-19T17:45:24.796815Z","shell.execute_reply.started":"2021-07-19T17:45:24.179479Z","shell.execute_reply":"2021-07-19T17:45:24.79565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve,auc,confusion_matrix,precision_score,recall_score,roc_auc_score\ny_score = clf.predict_proba(x_test)\nfpr, tpr, thresholds = roc_curve(y_test, y_score[:, 1])\nroc_auc_score = roc_auc_score(y_test,y_score[:,1])\nprint(roc_auc_score)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:24.798129Z","iopub.execute_input":"2021-07-19T17:45:24.798445Z","iopub.status.idle":"2021-07-19T17:45:25.075682Z","shell.execute_reply.started":"2021-07-19T17:45:24.798406Z","shell.execute_reply":"2021-07-19T17:45:25.074566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_probs = [0 for i in range(len(y_test))]\np_fpr, p_tpr, _ = roc_curve(y_test, random_probs, pos_label=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:25.076983Z","iopub.execute_input":"2021-07-19T17:45:25.077279Z","iopub.status.idle":"2021-07-19T17:45:25.206906Z","shell.execute_reply.started":"2021-07-19T17:45:25.077248Z","shell.execute_reply":"2021-07-19T17:45:25.205586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('seaborn')\nplt.plot(fpr,tpr,linestyle = '--',color = 'green',label='Decision Tree')\nplt.plot(p_fpr,p_tpr,linestyle='--',color = 'blue')\nplt.legend()\nplt.xlabel('FPR')\nplt.ylabel('TPR')\nplt.title('AUC-ROC Curve')","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:25.207983Z","iopub.execute_input":"2021-07-19T17:45:25.208268Z","iopub.status.idle":"2021-07-19T17:45:25.409604Z","shell.execute_reply.started":"2021-07-19T17:45:25.208241Z","shell.execute_reply":"2021-07-19T17:45:25.408063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = clf.predict(x_test)\nprecision = precision_score(y_test, y_pred, average = 'weighted')\nrecall = recall_score(y_test, y_pred, average = 'weighted')\nprint(\"Precision: %s, Recall: %s\" %(precision, recall))","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:25.411576Z","iopub.execute_input":"2021-07-19T17:45:25.412066Z","iopub.status.idle":"2021-07-19T17:45:25.832774Z","shell.execute_reply.started":"2021-07-19T17:45:25.412016Z","shell.execute_reply":"2021-07-19T17:45:25.83158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matrix = confusion_matrix(y_test,y_pred)\ntn, fp, fn, tp = matrix.ravel()\nprint(matrix)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:25.83395Z","iopub.execute_input":"2021-07-19T17:45:25.834259Z","iopub.status.idle":"2021-07-19T17:45:26.410868Z","shell.execute_reply.started":"2021-07-19T17:45:25.83423Z","shell.execute_reply":"2021-07-19T17:45:26.409466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#For example:\n#Here total_return and cost values are examples\ntotal_return = 0.2\ncost = 0.05\ntotal_return = tp * total_return\ntotal_cost = (tp + fp) * cost \nroi = total_return / total_cost\nprint(\"Total return: %s, Total cost: %s, ROI: %s\" %(\n  total_return, total_cost, roi))","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:26.415521Z","iopub.execute_input":"2021-07-19T17:45:26.415851Z","iopub.status.idle":"2021-07-19T17:45:26.422239Z","shell.execute_reply.started":"2021-07-19T17:45:26.41582Z","shell.execute_reply":"2021-07-19T17:45:26.421071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for max_depth_val in [2, 3, 5, 10, 15, 20]:\n    clf = DecisionTreeClassifier(max_depth = max_depth_val)\n    print(\"Evaluating tree with max_depth = %s\" %(max_depth_val))\n    y_pred = clf.fit(x_train,y_train).predict(x_test) \n    print(\"Confusion matrix: \")\n    print(confusion_matrix(y_test, y_pred))\n    prec = precision_score(y_test, y_pred, average = 'weighted')\n    recall = recall_score(y_test, y_pred, average = 'weighted')\n    print(\"Precision: %s, Recall: %s\" %(prec, recall))","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:45:26.424422Z","iopub.execute_input":"2021-07-19T17:45:26.424842Z","iopub.status.idle":"2021-07-19T17:46:26.861867Z","shell.execute_reply.started":"2021-07-19T17:45:26.424796Z","shell.execute_reply":"2021-07-19T17:46:26.860656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold,cross_val_score\n# Set up k-fold\nk_fold = KFold(n_splits = 5)\n\n# Evaluate precision and recall for each fold\nprecision = cross_val_score(\n  clf, x_train, y_train, cv = k_fold, scoring = 'precision_weighted')\nrecall = cross_val_score(\n  clf, x_train, y_train, cv = k_fold, scoring = 'recall_weighted')\nprint(\"Precision scores: %s\" %(precision)) \nprint(\"Recall scores: %s\" %(recall))\nprint(k_fold)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:46:26.863391Z","iopub.execute_input":"2021-07-19T17:46:26.863673Z","iopub.status.idle":"2021-07-19T17:48:40.98239Z","shell.execute_reply.started":"2021-07-19T17:46:26.863645Z","shell.execute_reply":"2021-07-19T17:48:40.98122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for max_depth_val in [3, 5, 10]:\n    k_fold = KFold(n_splits = 4)\n    clf = DecisionTreeClassifier(max_depth = max_depth_val)\n    print(\"Evaluating Decision Tree for max_depth = %s\" %(max_depth_val))\n    y_pred = clf.fit(x_train, y_train).predict(x_test) \n  \n    cv_precision = cross_val_score(clf, x_train, y_train, cv = k_fold, scoring = 'recall_weighted')\n    precision = recall_score(y_test, y_pred, average = 'weighted')\n    print(\"Cross validation Recall: %s\" %(cv_precision))\n    print(\"Test Recall: %s\" %(precision))\n  ","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:48:40.984148Z","iopub.execute_input":"2021-07-19T17:48:40.984639Z","iopub.status.idle":"2021-07-19T17:50:08.482251Z","shell.execute_reply.started":"2021-07-19T17:48:40.984586Z","shell.execute_reply":"2021-07-19T17:50:08.481041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import RandomizedSearchCV\n\nparam_grid = {\n    'bootstrap': [True],\n    'max_depth': [10], \n    'n_estimators':[2,5,10,20,50],\n    'min_samples_split': [2, 3, 4],\n    'max_features' : ['log2']\n}\n\nrf = RandomForestClassifier(random_state=42)\n\nrf_search = RandomizedSearchCV(estimator = rf, param_distributions=param_grid, \n                          cv = 5, n_jobs = -1, verbose = 2, n_iter = 10)\n\nmodel = rf_search.fit(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T17:50:08.483759Z","iopub.execute_input":"2021-07-19T17:50:08.484227Z","iopub.status.idle":"2021-07-19T18:02:11.297189Z","shell.execute_reply.started":"2021-07-19T17:50:08.484165Z","shell.execute_reply":"2021-07-19T18:02:11.295879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2021-07-19T18:02:11.298961Z","iopub.execute_input":"2021-07-19T18:02:11.299378Z","iopub.status.idle":"2021-07-19T18:02:11.306566Z","shell.execute_reply.started":"2021-07-19T18:02:11.299319Z","shell.execute_reply":"2021-07-19T18:02:11.30548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_rf = RandomForestClassifier(n_estimators= 50,\n min_samples_split= 2,\n max_features= 'log2',\n max_depth= 10,\n bootstrap= True,\n random_state=42)\nclf_rf.fit(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2021-07-19T18:13:36.390183Z","iopub.execute_input":"2021-07-19T18:13:36.390734Z","iopub.status.idle":"2021-07-19T18:13:36.469331Z","shell.execute_reply.started":"2021-07-19T18:13:36.390637Z","shell.execute_reply":"2021-07-19T18:13:36.467879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_pred = clf_rf.predict(x_test)\n\nprint(confusion_matrix(y_test, y_test_pred))\nprec = precision_score(y_test, y_test_pred, average = 'weighted')\nrecall = recall_score(y_test, y_test_pred, average = 'weighted')\nprint(\"Precision: %s, Recall: %s\" %(prec, recall))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import export_graphviz\nfrom six import StringIO\nimport pydot as pydot\nimport graphviz\nfrom IPython.display import Image\n\ndot_data = export_graphviz(clf,out_file=None,filled=True,rounded=True,special_characters=True)\ngraph = graphviz.Source(dot_data)\ngraph\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}