{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1><center>Machine Learning Project - AVAZU<center></h1>\n","metadata":{"id":"-3rW8oUMwRyd"}},{"cell_type":"code","source":"import sys\nfrom zipfile import ZipFile\nimport os.path as op\ntry:\n    from urllib.request import urlretrieve\nexcept ImportError:  # Python 2 compat\n    from urllib import urlretrieve\n\nAVAZU_URL = \"https://bianchi.wp.imt.fr/files/2019/05/train-1000000.zip\"\nAVAZU_FILENAME = AVAZU_URL.rsplit('/', 1)[1]\n\n\nif not op.exists(AVAZU_FILENAME):\n    print('Downloading %s to %s...' % (AVAZU_URL, AVAZU_FILENAME))\n    urlretrieve(AVAZU_URL, AVAZU_FILENAME)\n    ZipFile(AVAZU_FILENAME).extractall('.')","metadata":{"id":"mCOAoM9HwRye","outputId":"3eca17d5-ed78-415e-d33e-0c4f109b77b8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install category_encoders","metadata":{"id":"yw64YGmyTcL_","outputId":"d00dc94c-7d78-4c44-ba5b-6ed15e5a25f4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U imbalanced-learn\n","metadata":{"id":"FX718171BJr9","outputId":"11fea831-4715-42b1-dbb3-7d12fafa2d84"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZipFile(AVAZU_FILENAME).extractall('.')","metadata":{"id":"CV31drqmt2kr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nfrom imblearn.under_sampling import TomekLinks\n%matplotlib inline","metadata":{"id":"NidSqTnpwRyo"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data description here: https://www.kaggle.com/c/avazu-ctr-prediction/data\n","metadata":{"id":"RDR5E716wRyr"}},{"cell_type":"code","source":"df = pd.read_csv('train-1000000')","metadata":{"id":"4rgQ-Mftwgeo"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Work assignment\n\n1.   Perform Exploratory Data Analysis (EDA)\n2.   Predict the probability that an add is clicked, based on the features.\n3.   Evaluate the performance of your method.\n\n##Work evaluation\n\nYour work will be evaluated based on:\n\n*   The interest of the EDA (4 points)\n*   The quality of the comments/remarks throughout the notebook (4 points)\n*   The feature engineering (5 points)\n*   The efforts made in trying several predictors and tuning them (5 points)\n*   The relevance of the performance criteria used to conclude (2 points)\n*   BONUS (+1pt) : Register to the AVAZU challenge in Kaggle. What is your rank?\n\nYou may find the following script useful to convert the raw time stamp into a more useable feature. For instance, you may generate new pandas series using the code:\n\n```\ndf['hour'].apply(lambda x:datesplit(x).weekday())\ndf['hour'].apply(lambda x:datesplit(x).hour)\n```\n\n","metadata":{"id":"yhZ_xDwOwRzC"}},{"cell_type":"code","source":"#@title Exploratory Data Analysis\n#observe the data\ndf.head(10)","metadata":{"id":"vstjF4ScthyH","outputId":"ac58834e-9f93-43d9-8e13-0cc538240c0b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use info() to summarize the dataframe\ndf.info()\n#apparently, we don't have null values in this dataset","metadata":{"id":"IFQmkSqPtkOH","outputId":"2d7aadaa-46d0-49a6-e0c4-d0a09a9b54e9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape\n#examine the shape the dataset--we have 24 features","metadata":{"id":"oQ_YLZCB4eHn","outputId":"166acf63-6917-42ee-aade-568ac663b5e0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check the labels distribution\nprint(df.click.value_counts())\nprint(\"\\n\")\nplt.figure()\nsns.countplot(x='click', data=df)\nplt.show()\n#as we can observe, most of the instances end up without getting a click\n#and we can see that the labels are very imbalanced between the 2 classes:\n#non-click outnumbered click\n#we will keep this in mind and deal with this later","metadata":{"id":"5NrZd7Hx1CvW","outputId":"83b1e935-1f16-4901-9429-097256335302"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#preprocess the data\n#drop the id column since it does not help with the model training\n#also drop feature\ndf = df.drop(columns=['id'])","metadata":{"id":"17MLrjABtkYH"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check the Data Type of the features\n\nnumerical = []\ncategorical = []\n\nfor col in (df.columns):\n    if df[col].dtype == \"object\":\n        categorical.append(col)\n    else:\n        numerical.append(col)\nprint(\"numerical features = \",numerical)\nprint(\"\\ncategorical features = \",categorical)","metadata":{"id":"2qUrpG9J43Yn","outputId":"c10e1f08-d00f-47b6-b25c-b7944eecc742"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check number of unique values under each feature\ndf.nunique()","metadata":{"id":"uPfwzVUhth0P","outputId":"936df8e3-38b4-4e2e-a3bb-d812eeb113cc"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Apprently, with our features, although being categorical, features such as \"app_id\", \"site_id\",\"app_domain\",\"C14\",\"C17\", \"hour\" and other features have too many unique values (high cardinality), and one-hot-coding won't be a practical option to deal with these columns.","metadata":{"id":"62SnORhB3zcF"}},{"cell_type":"code","source":"#calculate pairwise correlation between each feature and see if we can\n#do something to the data\n\ncorr = df.corr()\nf, ax = plt.subplots(figsize=(10,10))\nsns.heatmap(corr, ax=ax, annot=True)\nplt.title(\"Pairwise Correlations\")","metadata":{"id":"MRmwR9jbth2n","outputId":"b3f79732-fa2c-4147-9eb1-da074fda2849"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above heatmap, we can observe that C1 and banner position are positively correlated, device type and banner position are positively related, device type and C1 are strongly positively related. C14 and C21 are also positively correlated. These several features are the ones that are positively correlated at a strong level. We inspect the negatively correlated features, and find that C18 and C17 are negatively correlated, C14 and C19 are negatively correlated, C1 and C15 are negatively correlated,C14 and C17, C17 and C21 are strongly correlated, C21 and C18 are strongly negatively correlated. We will drop some strongly correlated features to avoid probelms, since they may cause overfitting or redundancy (so we can simplify the model).","metadata":{"id":"dAm2DbhK415e"}},{"cell_type":"code","source":"#manually check the composition of certain features\nprint(df['app_category'].value_counts(normalize = True))","metadata":{"id":"TruLiFUC7hQa","outputId":"5f69b94f-3401-42c2-9830-2cf000bb3fc3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['app_domain'].value_counts(normalize = True))","metadata":{"id":"2p6tdfgf7hTa","outputId":"81185f37-8bbe-4331-f059-d48430d752c4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['app_id'].value_counts(normalize = True))","metadata":{"id":"oblD2Yh_7hWT","outputId":"4ac8e638-a77f-4ed8-e184-a2b5d3f51981"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['site_category'].value_counts(normalize = True))","metadata":{"id":"j-bMHU-Q7hY5","outputId":"339bba42-b241-4cb7-a286-ebb176b195d6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['site_domain'].value_counts(normalize = True))","metadata":{"id":"EHKYOEOa7hbs","outputId":"d7ae6dbc-f94c-4191-b8d3-818dc76e53a0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['site_id'].value_counts(normalize = True))","metadata":{"id":"IQfzriWs7heN","outputId":"18a22ace-e258-421b-8f87-575f21c8b413"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['device_id'].value_counts(normalize = True))","metadata":{"id":"QQrNDppZ7hgr","outputId":"2ec51abc-67d6-431d-cd3f-b972c873997e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['device_ip'].value_counts(normalize = True))","metadata":{"id":"5rfY2Vo77hiq","outputId":"ee287ee3-fcb0-44a3-8112-df5440e2698d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['device_model'].value_counts(normalize = True))","metadata":{"id":"-RoFEOYJ8opo","outputId":"612eff47-5a32-42a7-e15c-e3fa2fa0d461"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df['C20'].value_counts(normalize = True))\n#the amount of '-1' seems to be suspicious--more like nan here comparing to other entries\n#we drop this feature due to that","metadata":{"id":"8TUQse3s8oso","outputId":"f0a5244e-65d9-41f5-9290-f9cbb9eb88d0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#do some visualiztion to better understand the composition\ndf.banner_pos.value_counts()","metadata":{"id":"76SOwfsNqH0R","outputId":"c6ad53c5-454d-4944-bed6-e803d651c110"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,10))#visualize banner position regarding banner_pos and click\nsns.countplot(x='banner_pos',hue='click',data=df)\n#we observe that most advertisment is in position 0 and 1","metadata":{"id":"my657_SlqNEA","outputId":"97505d89-1a42-41a1-fcf8-1b887abddb7a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#do the same for C1 and click (label)\ndf.banner_pos.value_counts()","metadata":{"id":"jjKfUjBLrBuL","outputId":"9c8ef56a-a22d-4f53-f81f-7de9c5f044d6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,10))#visualize banner position regarding banner_pos and click\nsns.countplot(x='C1',hue='click',data=df)\n#we observe that most advertisment has C1 of 1005, and it's also when most label 1\n#happens","metadata":{"id":"HaMjke8RrG2d","outputId":"8ce1304b-232c-44f6-968b-02904fab8895"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas.core.groupby.groupby import Categorical\n#@title Feature Engineering\n#training data\n#categorical features\n#labels\n#we drop the C20 column as previously mentioned\ndf = df.drop(columns=['C20'])#drop C20\ndf_y = df['click'] #labels\ndf = df.drop(columns = ['click'])\nnumerical.remove('C20')\n\n","metadata":{"id":"PNrfF2d5th5e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.nunique() #we check the data again","metadata":{"id":"cuGITH4w_spG","outputId":"a1819742-fe16-4394-9bc9-11d038071d8e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#preprocess HOUR data--\ndf['time'] = df['hour'].astype(str).str[-2:].astype(int)\ndf['day'] = df['hour'].astype(str).str[-4:-2].astype(int)","metadata":{"id":"cISewLbVUiO3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop the old 'hour' column\ndf = df.drop(columns=['hour'])","metadata":{"id":"4WIj1XvxUiRm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we check the heatmap again to confirm which features to drop\ncorr = df.corr()\nf, ax = plt.subplots(figsize=(10,10))\nsns.heatmap(corr, ax=ax, annot=True)\nplt.title(\"Pairwise Correlations\")","metadata":{"id":"1bwA_q1wGBiX","outputId":"0af4b5ac-6164-4e3d-ffb2-c57871548d68"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from the heat map, we can see that\n#C1 and device_type are highly corelated so we drop C1 to avoid repetition\n#and overfitting; also since C1 is anonymous, so it's better to drop it and\n#keep device_type same for device_type and banner_pos\n#and avoid having an overly high data dimension\n#so for C21-C18, we drop C21\n#C14 and C17 are highly co-related, arbitarily dropping C14 out of the same\n#reason\n\ndf_x = df.drop(columns = ['C14', 'C1','device_type','C21'])","metadata":{"id":"A-38CD9qWMAI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we already knew the data is very unbalanced,\n#in this case, we first sepearte the data to training and testing sets\n#and see how imbalanced the y-labels are\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(df_x, df_y, test_size=0.3, random_state=42)\nprint(\"Before Undersampling:\")\nprint(\"Number of instances in each class:\", np.bincount(y_train))\n#therefore, we can see that the y-train is very disparate, with\n#580947 of label 0, and 119053 of label 1\n#we will adjust this in our training data to improve the model","metadata":{"id":"8vX5YkPnUiWj","outputId":"69093214-bed5-4077-c920-dc9877c94dfd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_x.nunique()#check the data","metadata":{"id":"EXGL8umNOp01","outputId":"084dd1c4-8d55-4233-b99a-73db1bd2742f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#do one hot coding for some low cardinality columns first\nfrom sklearn.preprocessing import OneHotEncoder\n\nlow_card_cols = ['banner_pos', 'device_conn_type','C15','C16','C18','day']\nohe = OneHotEncoder(sparse=False, handle_unknown='ignore')\nX_train_mid = ohe.fit_transform(X_train[low_card_cols])\nX_test_mid = ohe.transform(X_test[low_card_cols])\n","metadata":{"id":"TiyRVNZpOg0T","outputId":"c963b337-c3ba-4fd7-ead6-760851ee3b0e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#target encoder\n#train and fit a target encoder\n#to solve the high cardinality problem within the data\nfrom category_encoders import TargetEncoder\ntarget_encoder = TargetEncoder()\nhigh_card_cols = ['site_id', 'site_domain','site_category','app_id','app_domain',\n'app_category','device_id','device_ip','device_model','C17','C19','time']\nX_train_target = target_encoder.fit_transform(X_train[high_card_cols], y_train)\nX_test_target = target_encoder.transform(X_test[high_card_cols])","metadata":{"id":"8otNE31narsb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#concatenate the 2 dataframes together\n#first convert numpy array to dataframe\nX_train_low_card_df = pd.DataFrame(X_train_mid, columns = ohe.get_feature_names_out(low_card_cols), index=X_train.index)\nX_test_low_card_df = pd.DataFrame(X_test_mid, columns = ohe.get_feature_names_out(low_card_cols), index=X_test.index)\n##\nX_train_processed = pd.concat([X_train_low_card_df, X_train_target], axis=1)\nX_test_processed = pd.concat([X_test_low_card_df, X_test_target], axis=1)\n","metadata":{"id":"A1BlA5SRVnq6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from imblearn.under_sampling import NearMiss\n#use NearMiss under sampling to resolve the imbalance problem\n#this select examples based on the distance to the nearest\n#examples in the majority class\n#if we do not do this, the model may have high accuracy, yet it\n#is problematic because the labels are very imbalanced, so by\n#even guess, it can give high accuracy, and it does not represents\n#the quality of the models\nnm = NearMiss()\n\nX_train_final, y_train_final = nm.fit_resample(X_train_processed, y_train)\n\nprint(\"After Undersampling:\")\nprint(\"Number of instances in each class:\", np.bincount(y_train_final))\n\n#Check the undersampled data\n#now it's balanced!","metadata":{"id":"DngtWbn3HVlz","outputId":"6afe71cb-81b1-4440-9a61-d7445d0119c3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@title Decision Tree\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.model_selection import GridSearchCV\n\n#the parameter grid\nparam_grid = {\n    'criterion': ['gini'],\n    'max_depth': [30, 40, 50],\n    'min_samples_split': [15,20,25],\n    'min_samples_leaf': [2, 5,10]\n}\n\n\ndtree = DecisionTreeClassifier(random_state=42)\ngrid_search = GridSearchCV(estimator = dtree, param_grid = param_grid,\n                           cv = 10, n_jobs = -1, verbose = 2)\n\n#fit the grid search\ngrid_search.fit(X_train_final, y_train_final)\n\n#best parameters\nbest_params = grid_search.best_params_\nprint(best_params)\n#with the best parameters\nbest_dtree = DecisionTreeClassifier(**best_params, random_state=42)\nbest_dtree.fit(X_train_final, y_train_final)\npredictions = best_dtree.predict(X_test_processed)\n\n","metadata":{"id":"OPTivGViZs3o","outputId":"a1cc3821-26b2-486b-8ac1-2495e521fee1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, precision_score, recall_score, roc_auc_score, roc_curve\nimport matplotlib.pyplot as plt\n\ny_pred = best_dtree.predict(X_test_processed)\ny_proba = best_dtree.predict_proba(X_test_processed)[:, 1]\nthreshold = 0.7  #set the threshold to be 0.7 since it's under sampling\n#increasing the threshold will typically increase precision but reduce recall\n#predictions using the threshold\ny_pred = (y_proba >= threshold).astype('int')\n#calculate metrics\naccuracy = accuracy_score(y_test, y_pred)\nprecision = precision_score(y_test, y_pred)\nrecall = recall_score(y_test, y_pred)\nroc_auc = roc_auc_score(y_test, y_proba)\n\nprint(f\"Accuracy: {accuracy}\")\nprint(f\"Precision: {precision}\")\nprint(f\"Recall: {recall}\")\nprint(f\"ROC AUC: {roc_auc}\")\n\n#ROC curve\nfpr, tpr, _ = roc_curve(y_test, y_proba)\nplt.figure(figsize=(8, 6))\nplt.plot(fpr, tpr, label=f'ROC curve (area = {roc_auc:.2f})')\nplt.plot([0, 1], [0, 1], color='navy', linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('ROC Decision Tree')\nplt.legend(loc=\"lower right\")\nplt.show()\n\n","metadata":{"id":"NxfMILXUZtBP","outputId":"da46a0e6-bb9b-442b-a746-987977bbad92"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@title Random Forest\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import GridSearchCV\n\n#Hyperparameters tuning\nrf = RandomForestClassifier(random_state=42)\nparam_grid = {\n    'n_estimators': [100,125,150],\n    'max_depth' : [9,10,11],\n    'criterion' :['gini']\n}\nCV_rf = GridSearchCV(estimator=rf, param_grid=param_grid, cv=10)\nCV_rf.fit(X_train_final, y_train_final)\n\n#best parameters\nprint(\"Best parameters: \", CV_rf.best_params_)\n\n#predict using the model with best parameters\nbest_rf = RandomForestClassifier(n_estimators=CV_rf.best_params_['n_estimators'],\n                                 max_depth=CV_rf.best_params_['max_depth'],\n                                 criterion=CV_rf.best_params_['criterion'],\n                                 random_state=42)\nbest_rf.fit(X_train_final, y_train_final)\n#increasing the threshold will typically increase precision but reduce recall\ny_pred_proba_rf = best_rf.predict_proba(X_test_processed)[:, 1]\n#predictions using the threshold\ny_pred_rf = (y_pred_proba_rf >= threshold).astype('int')\n#evaluation\naccuracy = accuracy_score(y_test, y_pred_rf)\nprecision = precision_score(y_test, y_pred_rf)\nrecall = recall_score(y_test, y_pred_rf)\nroc_auc = roc_auc_score(y_test, y_pred_rf)\n\nprint('Accuracy:', accuracy)\nprint('Precision:', precision)\nprint('Recall:', recall)\nprint('ROC AUC:', roc_auc)","metadata":{"id":"IojB3EC8Uiby","outputId":"d7398016-9419-4153-be14-8efef9667b89"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualization\nfrom sklearn.metrics import roc_curve, auc\nimport matplotlib.pyplot as plt\n\ny_scores_rf = best_rf.predict_proba(X_test_processed)[:, 1]\nfpr_rf, tpr_rf, _ = roc_curve(y_test, y_scores_rf)\nroc_auc_rf = auc(fpr_rf, tpr_rf)\nplt.figure()\nlw = 2\nplt.plot(fpr_rf, tpr_rf, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc_rf)\nplt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('FPR')\nplt.ylabel('TPR')\nplt.title('RANDOM FOREST ROC')\nplt.legend(loc=\"lower right\")\nplt.show()\n","metadata":{"id":"bBGExHxywTKh","outputId":"b4aa3208-aa28-4a56-c55b-d1e3ad6a0863"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#@title Logistic Regression\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import StandardScaler #for logistic regression,\n#we also want to re-scale the data\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train_final)\nX_test_scaled = scaler.transform(X_test_processed)\n#logistic regression\n#tuning\nlogreg = LogisticRegression(max_iter=1000, random_state=42)\nparam_grid = {\n    'C': [0.05, 0.1, 1, 10],\n    'penalty': ['l2'],\n}\nCV_logreg = GridSearchCV(estimator=logreg, param_grid=param_grid, cv=10)\nCV_logreg.fit(X_train_scaled, y_train_final)\n\n#best parameters\nprint(\"Best parameters: \", CV_logreg.best_params_)\n\n#predict using the model with best parameters\nbest_logreg = LogisticRegression(C=CV_logreg.best_params_['C'],\n                                 penalty=CV_logreg.best_params_['penalty'],\n                                 max_iter=1000,\n                                 random_state=42)\nbest_logreg.fit(X_train_scaled, y_train_final)\n#y_pred_logreg = best_logreg.predict(X_test_scaled)\n#####\n\n#increasing the threshold will typically increase precision but reduce recall\ny_pred_proba_logreg = best_logreg.predict_proba(X_test_scaled)[:, 1]\n#predictions using the threshold\ny_pred_logreg = (y_pred_proba_logreg >= threshold).astype('int')\n#####\n#evaluation\naccuracy = accuracy_score(y_test, y_pred_logreg)\nprecision = precision_score(y_test, y_pred_logreg)\nrecall = recall_score(y_test, y_pred_logreg)\nroc_auc = roc_auc_score(y_test, y_pred_logreg)\n\nprint('Accuracy:', accuracy)\nprint('Precision:', precision)\nprint('Recall:', recall)\nprint('ROC AUC:', roc_auc)\n","metadata":{"id":"tlaE46P1UieQ","outputId":"7cf61de5-6710-4bc4-af2a-ef6c3951aecb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualization\nfrom sklearn.metrics import roc_curve, auc\nimport matplotlib.pyplot as plt\n\ny_scores_rf = best_logreg.predict_proba(X_test_scaled)[:, 1]\nfpr_rf, tpr_rf, _ = roc_curve(y_test, y_scores_rf)\nroc_auc_rf = auc(fpr_rf, tpr_rf)\nplt.figure()\nlw = 2\nplt.plot(fpr_rf, tpr_rf, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc_rf)\nplt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('FPR')\nplt.ylabel('TPR')\nplt.title('LOGISTIC REGRESSION ROC')\nplt.legend(loc=\"lower right\")\nplt.show()","metadata":{"id":"MGIYk281zDkF","outputId":"a01409e9-9066-481f-8229-ca32d5857ed3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@title XGBoost\nimport xgboost as xgb\n\n#tuning\nxgboost = xgb.XGBClassifier(random_state=42)\nparam_grid = {\n    'n_estimators': [50,75,100],\n    'learning_rate': [0.01, 0.1, 0.2],\n    'max_depth': [3,4,5]\n}\nCV_xgb = GridSearchCV(estimator=xgboost, param_grid=param_grid, cv=10)\nCV_xgb.fit(X_train_final, y_train_final)\n\n#best parameters\nprint(\"Best parameters: \", CV_xgb.best_params_)\n\n#predict using the model with best parameters\nbest_xgb = xgb.XGBClassifier(n_estimators=CV_xgb.best_params_['n_estimators'],\n                             learning_rate=CV_xgb.best_params_['learning_rate'],\n                             max_depth=CV_xgb.best_params_['max_depth'],\n                             random_state=42)\nbest_xgb.fit(X_train_final, y_train_final)\ny_pred_proba_xgb = best_xgb.predict_proba(X_test_processed)[:, 1]\n#predictions using the threshold\ny_pred_xgb = (y_pred_proba_xgb >= threshold).astype('int')\n#evaluation\naccuracy = accuracy_score(y_test, y_pred_xgb)\nprecision = precision_score(y_test, y_pred_xgb)\nrecall = recall_score(y_test, y_pred_xgb)\nroc_auc = roc_auc_score(y_test, y_pred_xgb)\n\nprint('Accuracy:', accuracy)\nprint('Precision:', precision)\nprint('Recall:', recall)\nprint('ROC AUC:', roc_auc)\n","metadata":{"id":"t-GgarbgUii-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualization\ny_scores_rf = best_xgb.predict_proba(X_test_processed)[:, 1]\nfpr_rf, tpr_rf, _ = roc_curve(y_test, y_scores_rf)\nroc_auc_rf = auc(fpr_rf, tpr_rf)\nplt.figure()\nlw = 2\nplt.plot(fpr_rf, tpr_rf, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc_rf)\nplt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('FPR')\nplt.ylabel('TPR')\nplt.title('XGB ROC')\nplt.legend(loc=\"lower right\")\nplt.show()","metadata":{"id":"QsgJbpMDUilm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@title Confusion Matrix for Every Model\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\n\n#create the confusion matrix for the Decision Tree\ncm = confusion_matrix(y_test, y_pred)\n\n#plot the confusion matrix\nplt.figure(figsize=(10,7))\nsns.heatmap(cm, annot=True, cmap='Blues', fmt='g')\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.title('Confusion Matrix for Decision Tree')\nplt.show()\n#True Negative: 129130, True Positive: 10433,\n#False Negative: 23489, False Positive:36948","metadata":{"id":"ywhuoCTIUivC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create the confusion matrix for Random Forest\ncm = confusion_matrix(y_test, y_pred_rf)\n\n# Plot the confusion matrix\nplt.figure(figsize=(10,7))\nsns.heatmap(cm, annot=True, cmap='Blues', fmt='g')\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.title('Confusion Matrix for Random Forest')\nplt.show()\n#True Negative: 129130, True Positive: 10433,\n#False Negative: 23489, False Positive:36948","metadata":{"id":"3skLAG0OUixi"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create the confusion matrix for Logistic Regression\ncm = confusion_matrix(y_test, y_pred_logreg)\n\n# Plot the confusion matrix\nplt.figure(figsize=(10,7))\nsns.heatmap(cm, annot=True, cmap='Blues', fmt='g')\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.title('Confusion Matrix for Logistic Regression')\nplt.show()\n#True Negative: 129130, True Positive: 10433,\n#False Negative: 23489, False Positive:36948","metadata":{"id":"f2zdLlxQUi0H"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create the confusion matrix for XGBoost\ncm = confusion_matrix(y_test, y_pred_xgb)\n\n# Plot the confusion matrix\nplt.figure(figsize=(10,7))\nsns.heatmap(cm, annot=True, cmap='Blues', fmt='g')\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.title('Confusion Matrix for XGBoost')\nplt.show()\n#True Negative: 129130, True Positive: 10433,\n#False Negative: 23489, False Positive:36948","metadata":{"id":"KKmY2SOQUi2Q"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Conclusion\noverall, we can see that the models didn't have very good accraucy\nin predicting the labels. We have low accuracy and precision, and low recall; this indicates that we have a high False Positive Rate and a high False Negative rate; in other words, we are not able to idenitify true positive cases and would misclassify them as negative, and we often misclassify true negative cases as positive. These may be caused by multiple problems, such as the imbalanced labels(even though we did undersampling to mitigate the problems) and the overly complicated, high-cardinality features. We may need to learn more about the nature of the data to select features (since right now they are anonymous) and then do\ntrain models on the properly processed dataset; a better oversampling/undersampling algorithm may be applied (currently not using SMOTE becasue of computational limits; but it might give better results then using Near Miss undersampling method). Overall, among all the 4 models, we might choose Logistic Regression as the best model and apply it to make predictions.","metadata":{"id":"CVsDvvHCNaQA"}},{"cell_type":"code","source":"","metadata":{"id":"B5MS_tHdUi4r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"U6wkzNRPUi7A"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"kzglbZxsUi9Y"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"BBDtA9L7Ui_m"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"cWUSgKASUjB2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"g-i2mCjhUisV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"Q-8WtxJ1UjEj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"Qf9GsrbjUjHI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"LaN8wn5pUjJy"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"jvc1HUn2UjLu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"DU_Ti3rMUjOc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"aee1NNeEUjQx"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"3vKXtnReUjTZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"oO3Zs9yhth7U"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"HZyY9rcTth9Z"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"YbAU202hth_u"},"execution_count":null,"outputs":[]}]}