{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Aim of the notebook. \nIn this notebook you'll observe the following points addressed. \n\n1. Importance of class balance in classification.\n2. Models with and without feature engineering to address the importance of feature engineering.\n3. Analysis of different ML models.\n4. Importance of cross-validation and maintaining the history.\n","metadata":{}},{"cell_type":"markdown","source":"# Loading requiered libraries.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\nsns.set_theme(style=\"darkgrid\")\npd.set_option('display.max_columns', 0)\nplt.style.use('ggplot')\npd.options.display.float_format = '{:.2f}'.format\nimport math\nfrom sklearn.preprocessing import LabelEncoder \nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.dummy import DummyClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom imblearn.over_sampling import RandomOverSampler\nfrom sklearn.tree import DecisionTreeClassifier\nfrom IPython.display import Image  \nfrom six import StringIO\nfrom sklearn.tree import export_graphviz\nimport graphviz\nfrom sklearn.ensemble import RandomForestClassifier\nimport statsmodels.api as sm\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom sklearn.feature_selection import RFE\nfrom sklearn.model_selection import GridSearchCV\nimport random\nimport gzip\nimport category_encoders as ce\npd.options.display.float_format = '{:.2f}'.format\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:51:25.929139Z","iopub.execute_input":"2023-01-30T18:51:25.930225Z","iopub.status.idle":"2023-01-30T18:51:28.098142Z","shell.execute_reply.started":"2023-01-30T18:51:25.930102Z","shell.execute_reply":"2023-01-30T18:51:28.097104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading input data and basic analysis","metadata":{}},{"cell_type":"code","source":"# Reading the input data\n\nnum_records = 40428967\nsample_size = 5000000\nskip_values = sorted(random.sample(range(1,num_records), num_records - sample_size))\nparse_date = lambda val : pd.datetime.strptime(val, '%y%m%d%H')\n\ntrain = pd.read_csv(\"../input/avazu-ctr-prediction/train.gz\", parse_dates=['hour'], date_parser=parse_date,\\\n                     skiprows=skip_values)\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:51:28.099874Z","iopub.execute_input":"2023-01-30T18:51:28.100248Z","iopub.status.idle":"2023-01-30T18:56:00.034217Z","shell.execute_reply.started":"2023-01-30T18:51:28.100216Z","shell.execute_reply":"2023-01-30T18:56:00.032938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading the test data.\n\ntest = pd.read_csv('../input/avazu-ctr-prediction/test.gz', parse_dates=['hour'], date_parser=parse_date)\ntest.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:56:00.035518Z","iopub.execute_input":"2023-01-30T18:56:00.036613Z","iopub.status.idle":"2023-01-30T18:57:50.864007Z","shell.execute_reply.started":"2023-01-30T18:56:00.036574Z","shell.execute_reply":"2023-01-30T18:57:50.862610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission file for final score.\n\nsubmission = pd.read_csv('../input/avazu-ctr-prediction/sampleSubmission.gz')\nsubmission.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:57:50.866645Z","iopub.execute_input":"2023-01-30T18:57:50.867079Z","iopub.status.idle":"2023-01-30T18:57:53.689713Z","shell.execute_reply.started":"2023-01-30T18:57:50.867043Z","shell.execute_reply":"2023-01-30T18:57:53.687848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape of the data read. \n\nprint('Train dataset:',train.shape)\nprint('Test dataset:',test.shape)\nprint('Submission:',submission.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:57:53.691662Z","iopub.execute_input":"2023-01-30T18:57:53.692142Z","iopub.status.idle":"2023-01-30T18:57:53.703448Z","shell.execute_reply.started":"2023-01-30T18:57:53.692104Z","shell.execute_reply":"2023-01-30T18:57:53.702137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hour column contains event date with all the details, extracting the same to create different columns.\n\ntrain['month'] = train['hour'].dt.month\ntrain['dayofweek'] = train['hour'].dt.dayofweek\ntrain['day'] = train['hour'].dt.day\ntrain['hour_time'] = train['hour'].dt.hour\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:57:53.704672Z","iopub.execute_input":"2023-01-30T18:57:53.705159Z","iopub.status.idle":"2023-01-30T18:57:55.899957Z","shell.execute_reply.started":"2023-01-30T18:57:53.705119Z","shell.execute_reply":"2023-01-30T18:57:55.898221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking the sum of the null values across all cloumns and rows.\n\ntrain['hour'].isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:57:55.902784Z","iopub.execute_input":"2023-01-30T18:57:55.903431Z","iopub.status.idle":"2023-01-30T18:57:55.927387Z","shell.execute_reply.started":"2023-01-30T18:57:55.903360Z","shell.execute_reply":"2023-01-30T18:57:55.925756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# info on each column of training dataset.\n\ntrain.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:57:55.929479Z","iopub.execute_input":"2023-01-30T18:57:55.930092Z","iopub.status.idle":"2023-01-30T18:57:55.951246Z","shell.execute_reply.started":"2023-01-30T18:57:55.930051Z","shell.execute_reply":"2023-01-30T18:57:55.949819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# No null values found in the dataset.\n\ntrain.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:57:55.952873Z","iopub.execute_input":"2023-01-30T18:57:55.953410Z","iopub.status.idle":"2023-01-30T18:57:58.172003Z","shell.execute_reply.started":"2023-01-30T18:57:55.953359Z","shell.execute_reply":"2023-01-30T18:57:58.170505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# looks like following columns have outliers  C15, C16, C19, C21. \n\ntrain.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:57:58.176419Z","iopub.execute_input":"2023-01-30T18:57:58.176913Z","iopub.status.idle":"2023-01-30T18:58:02.336494Z","shell.execute_reply.started":"2023-01-30T18:57:58.176876Z","shell.execute_reply":"2023-01-30T18:58:02.334763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dealing with outliers by capping\n\ncol = ['C15', 'C16', 'C19', 'C21']\nfor col in col:\n    percentiles = train[col].quantile(0.98)\n    if train[col].quantile(0.98) < 0.5 * train[col].max():\n        train[col][train[col] >= percentiles] = percentiles","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:02.338712Z","iopub.execute_input":"2023-01-30T18:58:02.339155Z","iopub.status.idle":"2023-01-30T18:58:02.920048Z","shell.execute_reply.started":"2023-01-30T18:58:02.339118Z","shell.execute_reply":"2023-01-30T18:58:02.918501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# segrigating numerical and categorical variables.\n\nnumerical = []\ncategorical = []\n\nfor col in (train.columns):\n    if train[col].dtype == \"object\":\n        categorical.append(col)\n    else:\n        numerical.append(col)\nprint(\"numerical columns = \",numerical)\nprint(\"\\ncategorical columns = \",categorical)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:02.922265Z","iopub.execute_input":"2023-01-30T18:58:02.922954Z","iopub.status.idle":"2023-01-30T18:58:02.937985Z","shell.execute_reply.started":"2023-01-30T18:58:02.922895Z","shell.execute_reply":"2023-01-30T18:58:02.935845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y is the target variable, analysing the same.\n# 83% values are 0 and 17% values are 1. data is highly imbalance. \n\nprint(train.click.value_counts(normalize = True))\nprint(\"\\n\")\nplt.figure()\nsns.countplot(x='click', data=train)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:02.939878Z","iopub.execute_input":"2023-01-30T18:58:02.940490Z","iopub.status.idle":"2023-01-30T18:58:03.774679Z","shell.execute_reply.started":"2023-01-30T18:58:02.940442Z","shell.execute_reply":"2023-01-30T18:58:03.773127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analysing categorical variables.","metadata":{}},{"cell_type":"code","source":"print(\"unique counts of site_id\", len(train['site_id'].unique()))\nprint(\"----------------------\")\nprint(train['site_id'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:03.776386Z","iopub.execute_input":"2023-01-30T18:58:03.777014Z","iopub.status.idle":"2023-01-30T18:58:04.541000Z","shell.execute_reply.started":"2023-01-30T18:58:03.776968Z","shell.execute_reply":"2023-01-30T18:58:04.539675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique counts of site_domain\", len(train['site_domain'].unique()))\nprint(\"----------------------\")\nprint(train['site_domain'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:04.542949Z","iopub.execute_input":"2023-01-30T18:58:04.543527Z","iopub.status.idle":"2023-01-30T18:58:05.303840Z","shell.execute_reply.started":"2023-01-30T18:58:04.543488Z","shell.execute_reply":"2023-01-30T18:58:05.302159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique counts of site_category\", len(train['site_category'].unique()))\nprint(\"----------------------\")\nprint(train['site_category'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:05.306127Z","iopub.execute_input":"2023-01-30T18:58:05.306520Z","iopub.status.idle":"2023-01-30T18:58:06.019917Z","shell.execute_reply.started":"2023-01-30T18:58:05.306487Z","shell.execute_reply":"2023-01-30T18:58:06.018547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique counts of app_id\", len(train['app_id'].unique()))\nprint(\"----------------------\")\nprint(train['app_id'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:06.022125Z","iopub.execute_input":"2023-01-30T18:58:06.023463Z","iopub.status.idle":"2023-01-30T18:58:06.789699Z","shell.execute_reply.started":"2023-01-30T18:58:06.023410Z","shell.execute_reply":"2023-01-30T18:58:06.788203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique counts of app_domain\", len(train['app_domain'].unique()))\nprint(\"----------------------\")\nprint(train['app_domain'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:06.791001Z","iopub.execute_input":"2023-01-30T18:58:06.791461Z","iopub.status.idle":"2023-01-30T18:58:07.587345Z","shell.execute_reply.started":"2023-01-30T18:58:06.791415Z","shell.execute_reply":"2023-01-30T18:58:07.585501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique counts of app_category\", len(train['app_category'].unique()))\nprint(\"----------------------\")\nprint(train['app_category'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:07.590718Z","iopub.execute_input":"2023-01-30T18:58:07.591705Z","iopub.status.idle":"2023-01-30T18:58:08.349330Z","shell.execute_reply.started":"2023-01-30T18:58:07.591652Z","shell.execute_reply":"2023-01-30T18:58:08.347825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique counts of device_id\", len(train['device_id'].unique()))\nprint(\"----------------------\")\nprint(train['device_id'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:08.350676Z","iopub.execute_input":"2023-01-30T18:58:08.351041Z","iopub.status.idle":"2023-01-30T18:58:09.971890Z","shell.execute_reply.started":"2023-01-30T18:58:08.351011Z","shell.execute_reply":"2023-01-30T18:58:09.970491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique counts of device_ip\", len(train['device_ip'].unique()))\nprint(\"----------------------\")\nprint(train['device_ip'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:09.973521Z","iopub.execute_input":"2023-01-30T18:58:09.974145Z","iopub.status.idle":"2023-01-30T18:58:16.092384Z","shell.execute_reply.started":"2023-01-30T18:58:09.974110Z","shell.execute_reply":"2023-01-30T18:58:16.090585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique counts of device_model\", len(train['device_model'].unique()))\nprint(\"----------------------\")\nprint(train['device_model'].value_counts(normalize = True))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:16.094930Z","iopub.execute_input":"2023-01-30T18:58:16.095453Z","iopub.status.idle":"2023-01-30T18:58:17.039603Z","shell.execute_reply.started":"2023-01-30T18:58:16.095407Z","shell.execute_reply":"2023-01-30T18:58:17.038085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## observations :\n1. All categorical variables have lot of unique values in it, one hot encoding is not a scalable approach. \n2. we will go with label encoding and scaling approach is a better idea for this.","metadata":{}},{"cell_type":"code","source":"# The code below will plot histograms for all numerical columns \n\nn = 2\nplt.figure(figsize=[15,3*math.ceil(len(numerical)/n)])\n\nfor i in range(len(numerical)):\n    plt.subplot(math.ceil(len(numerical)/n),n,i+1)\n    sns.distplot(train[numerical[i]])\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T18:58:17.041355Z","iopub.execute_input":"2023-01-30T18:58:17.041913Z","iopub.status.idle":"2023-01-30T19:04:54.879059Z","shell.execute_reply.started":"2023-01-30T18:58:17.041872Z","shell.execute_reply":"2023-01-30T19:04:54.877817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## observations \n1. Y and Click looks like same columns, after co-relation we can drop on of them.\n2. month column has only 1 data entry, no exrtra information is added, can be dropped \n3. banner pos, device conn, C20, C15, C16 looks like data is cenetered around certain values. \n","metadata":{}},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:04:54.881026Z","iopub.execute_input":"2023-01-30T19:04:54.882414Z","iopub.status.idle":"2023-01-30T19:04:54.890331Z","shell.execute_reply.started":"2023-01-30T19:04:54.882360Z","shell.execute_reply":"2023-01-30T19:04:54.889452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pearson correlation table to find the relationship with output with all input features. \n\ncorr = train.corr()\nf, ax = plt.subplots(figsize=(10,10))\nsns.heatmap(corr, ax=ax, annot=True,linewidths=3,cmap='YlGn')\nplt.title(\"Pearson correlation of Features\", y=1.05, size=15)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:04:54.891961Z","iopub.execute_input":"2023-01-30T19:04:54.892690Z","iopub.status.idle":"2023-01-30T19:05:01.886918Z","shell.execute_reply.started":"2023-01-30T19:04:54.892652Z","shell.execute_reply":"2023-01-30T19:05:01.885626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## observation\n1. month has got no significance, better to drop it\n2. y and click are same drop click column \n3. C14 and C17 are highly co-related, later will remove one of them after the base model.\n4. device type with C1 are highly co-related, later will remove one of them after the base model.\n5. Removing C20 anomalised column, since it have got nearly 47% of values with -1. As a categorical variable\nit's not expected to have values as -1.","metadata":{}},{"cell_type":"code","source":"# as said above dropping columns. \n\ntrain.drop(['month', 'C20'], axis=1, inplace=True)\ntrain.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:01.888439Z","iopub.execute_input":"2023-01-30T19:05:01.888920Z","iopub.status.idle":"2023-01-30T19:05:02.771086Z","shell.execute_reply.started":"2023-01-30T19:05:01.888857Z","shell.execute_reply":"2023-01-30T19:05:02.769833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preperation\n","metadata":{}},{"cell_type":"code","source":"# id column have a unique columns so, keeping that don't proive any significance,hence dropping.\n# hour column have been derived into different columns, hence dropping. \n# rename click to y (output)\n# after dropping hour column, hour_time can be made as hour\n\ntrain.drop(['id', 'hour'], axis = 1, inplace = True) \ntrain.rename(columns={'click': 'y',\n                   'hour_time': 'hour'},\n          inplace=True, errors='raise')\n\ntrain.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:02.772511Z","iopub.execute_input":"2023-01-30T19:05:02.772981Z","iopub.status.idle":"2023-01-30T19:05:03.745936Z","shell.execute_reply.started":"2023-01-30T19:05:02.772940Z","shell.execute_reply":"2023-01-30T19:05:03.744599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset is huge and running multiple algo will take time and resources might exhaust, \n# hence taking only 10% of the data for analysis.\n\nsampled_data = train.sample(frac=0.1, random_state=42)\nX = sampled_data.drop(['y'], axis=1)\ny = sampled_data['y']","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:03.753921Z","iopub.execute_input":"2023-01-30T19:05:03.754990Z","iopub.status.idle":"2023-01-30T19:05:04.868058Z","shell.execute_reply.started":"2023-01-30T19:05:03.754929Z","shell.execute_reply":"2023-01-30T19:05:04.866919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After taking the sample of data, still the ratio of output remains same.\n\nprint(train.y.value_counts(normalize = True))\nprint(\"\\n\")\nplt.figure()\nsns.countplot(x=y)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:04.869454Z","iopub.execute_input":"2023-01-30T19:05:04.869970Z","iopub.status.idle":"2023-01-30T19:05:05.157400Z","shell.execute_reply.started":"2023-01-30T19:05:04.869918Z","shell.execute_reply":"2023-01-30T19:05:05.156428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_encoder = ce.TargetEncoder()\nX = target_encoder.fit_transform(X, y)\nX.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:05.158965Z","iopub.execute_input":"2023-01-30T19:05:05.159564Z","iopub.status.idle":"2023-01-30T19:05:18.880651Z","shell.execute_reply.started":"2023-01-30T19:05:05.159529Z","shell.execute_reply":"2023-01-30T19:05:18.879487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntarget_encoder = ce.TargetEncoder()\nfor col in (X.columns):\n    if X[col].dtype == \"object\":\n        X[col] = target_encoder.fit_transform(X[col], y)\n\nX.head(2)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:18.882081Z","iopub.execute_input":"2023-01-30T19:05:18.882466Z","iopub.status.idle":"2023-01-30T19:05:18.890979Z","shell.execute_reply.started":"2023-01-30T19:05:18.882434Z","shell.execute_reply":"2023-01-30T19:05:18.889651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size= 0.3, random_state= 42)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:18.892790Z","iopub.execute_input":"2023-01-30T19:05:18.893242Z","iopub.status.idle":"2023-01-30T19:05:19.388439Z","shell.execute_reply.started":"2023-01-30T19:05:18.893207Z","shell.execute_reply":"2023-01-30T19:05:19.387227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building basic models [no feature engineering]\n 1. in this no feature engineering is done apart from removing definate columns click and month.\n 2. We will run 3 models \n      a. Logistic regression [for explainability and finding linear relationship]\n      b. Decision tree classifier [for explainability and also for non-linear relation]\n      c. Random forest classifier [for accuracy and improving the model] \n 3. Target variable is highly imbalance, so model will be baised towards majority class. for every base model will try with both balance and imbalance data.","metadata":{}},{"cell_type":"markdown","source":"## defining functions for later use.","metadata":{}},{"cell_type":"code","source":"# this function helps in evaluation the given model and provide accuracy and confusion matrix. \n\ndef model_eval(model, X_test, y_test):\n    y_pred = model.predict(X_test)\n    accuracy = metrics.accuracy_score(y_test, y_pred)\n    #print(accuracy)\n    cnf_matrix = metrics.confusion_matrix(y_test, y_pred)\n    #print(cnf_matrix)\n    #metrics.plot_confusion_matrix(model, X_test, y_test)\n    #plt.show()\n    return accuracy, cnf_matrix","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:19.389951Z","iopub.execute_input":"2023-01-30T19:05:19.390869Z","iopub.status.idle":"2023-01-30T19:05:19.396801Z","shell.execute_reply.started":"2023-01-30T19:05:19.390830Z","shell.execute_reply":"2023-01-30T19:05:19.395454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plots the ROC curve and returns false positive rate, true positive rate, and thresholds. \n\ndef draw_roc(model, Xtest, actual):\n    probs = model.predict(X_test)\n    fpr, tpr, thresholds = metrics.roc_curve( actual, probs,\n                                              drop_intermediate = False )\n    auc_score = metrics.roc_auc_score( actual, probs )\n    plt.figure(figsize=(6, 6))\n    plt.plot( fpr, tpr, label='ROC curve (area = %0.2f)' % auc_score )\n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate or [1 - True Negative Rate]')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic example')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\n    return fpr, tpr, thresholds","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:19.398616Z","iopub.execute_input":"2023-01-30T19:05:19.399492Z","iopub.status.idle":"2023-01-30T19:05:19.418116Z","shell.execute_reply.started":"2023-01-30T19:05:19.399452Z","shell.execute_reply":"2023-01-30T19:05:19.416924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Returns the ctossvalidation score for given number of n folds. \n\ndef cross_val (model, x, y, folds):\n    crossvalidation = cross_val_score(model, x, y, cv= folds, n_jobs=-1)\n    return crossvalidation","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:19.419473Z","iopub.execute_input":"2023-01-30T19:05:19.420549Z","iopub.status.idle":"2023-01-30T19:05:19.437385Z","shell.execute_reply.started":"2023-01-30T19:05:19.420511Z","shell.execute_reply":"2023-01-30T19:05:19.436020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this stores the results in dataframe for evaluating the final result. \n\ndef store_results(name, ytrain, xtrain, ytest, xtest, model, folds):\n    \n    crossvalidation = cross_val(model, xtrain, ytrain, folds)\n    \n    accuracy_test, cm1 =  model_eval(model, xtrain, ytrain)\n    TP = cm1[1,1] # true positive \n    TN = cm1[0,0] # true negatives\n    FP = cm1[0,1] # false positives\n    FN = cm1[1,0] # false negatives\n    recall_test = TP / (TP+FP)\n    precision_test = TP / (TP+FN)\n    \n    accuracy_train, cm1 =  model_eval(model, xtest, ytest)\n    TP = cm1[1,1] # true positive \n    TN = cm1[0,0] # true negatives\n    FP = cm1[0,1] # false positives\n    FN = cm1[1,0] # false negatives\n    recall_train = TP / (TP+FP)\n    precision_train = TP / (TP+FN)\n\n    entry = {'Model': [name],\n          'Accuracy_train': [accuracy_train],\n          'recall_train': [recall_train],\n          'precision_train': [precision_train],\n          'Accuracy_test': [accuracy_train],\n          'recall_test': [recall_train],\n          'precision_test': [precision_train],\n          'CrossVal_Mean': [crossvalidation.mean()],           \n          'CrossVal1': [crossvalidation[0]],\n          'CrossVal2': [crossvalidation[1]],\n          'CrossVal3': [crossvalidation[2]],\n          'CrossVal4': [crossvalidation[3]],\n          'CrossVal5': [crossvalidation[4]],\n          }\n    result = pd.DataFrame(entry)\n    return result","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:19.438943Z","iopub.execute_input":"2023-01-30T19:05:19.439875Z","iopub.status.idle":"2023-01-30T19:05:19.453004Z","shell.execute_reply.started":"2023-01-30T19:05:19.439837Z","shell.execute_reply":"2023-01-30T19:05:19.451816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dummy classifier for verification\nDummy classifier helps in baseling the model performace w.r.t dominanat class. ","metadata":{"execution":{"iopub.status.busy":"2023-01-25T15:20:18.072975Z","iopub.execute_input":"2023-01-25T15:20:18.073372Z","iopub.status.idle":"2023-01-25T15:20:18.078413Z","shell.execute_reply.started":"2023-01-25T15:20:18.073340Z","shell.execute_reply":"2023-01-25T15:20:18.077385Z"}}},{"cell_type":"code","source":"outcome = pd.DataFrame()\ndummy_clf = DummyClassifier(strategy= \"most_frequent\")\ndummy_clf.fit(X_train, y_train)\naccuracy, cnf_matrix  = model_eval(dummy_clf, X_train, y_train)\nprint(accuracy)\nprint(cnf_matrix)\ntemp = store_results(\"Dummy classifier\", y_train, X_train, y_test, X_test, dummy_clf, 5)\noutcome = outcome.append(temp)\noutcome","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:19.454485Z","iopub.execute_input":"2023-01-30T19:05:19.455174Z","iopub.status.idle":"2023-01-30T19:05:22.033495Z","shell.execute_reply.started":"2023-01-30T19:05:19.455137Z","shell.execute_reply":"2023-01-30T19:05:22.032145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic regression model with imbalance precidtor ","metadata":{}},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:22.036005Z","iopub.execute_input":"2023-01-30T19:05:22.037403Z","iopub.status.idle":"2023-01-30T19:05:22.080117Z","shell.execute_reply.started":"2023-01-30T19:05:22.037347Z","shell.execute_reply":"2023-01-30T19:05:22.078825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = MinMaxScaler()\nX_train[X_train.columns] = scaler.fit_transform(X_train[X_train.columns])\nX_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:22.082009Z","iopub.execute_input":"2023-01-30T19:05:22.083368Z","iopub.status.idle":"2023-01-30T19:05:22.393748Z","shell.execute_reply.started":"2023-01-30T19:05:22.083299Z","shell.execute_reply":"2023-01-30T19:05:22.392435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LogisticRegression()\nmodel.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:22.395249Z","iopub.execute_input":"2023-01-30T19:05:22.395591Z","iopub.status.idle":"2023-01-30T19:05:29.923483Z","shell.execute_reply.started":"2023-01-30T19:05:22.395562Z","shell.execute_reply":"2023-01-30T19:05:29.921569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test[X_test.columns] = scaler.transform(X_test[X_test.columns])\nX_test.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:29.926157Z","iopub.execute_input":"2023-01-30T19:05:29.927425Z","iopub.status.idle":"2023-01-30T19:05:30.100695Z","shell.execute_reply.started":"2023-01-30T19:05:29.927349Z","shell.execute_reply":"2023-01-30T19:05:30.099547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_test)\naccuracy, cnf_matrix = model_eval(model, X_test, y_test)\nprint(accuracy)\nprint(cnf_matrix)\ntemp = store_results(\"logistic regres - imbalance predict\", y_train, X_train, y_test, X_test, model, 5)\noutcome = outcome.append(temp)\noutcome","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:30.102141Z","iopub.execute_input":"2023-01-30T19:05:30.102480Z","iopub.status.idle":"2023-01-30T19:05:40.829511Z","shell.execute_reply.started":"2023-01-30T19:05:30.102449Z","shell.execute_reply":"2023-01-30T19:05:40.828082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_roc(model, X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:40.831067Z","iopub.execute_input":"2023-01-30T19:05:40.831439Z","iopub.status.idle":"2023-01-30T19:05:41.223153Z","shell.execute_reply.started":"2023-01-30T19:05:40.831405Z","shell.execute_reply":"2023-01-30T19:05:41.221776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observation\n1. Logistic regression is behaving almost same as dummy classifier.\n2. ROC_AUC isn't doing better at all","metadata":{}},{"cell_type":"markdown","source":"## Basic regression model with balance precidtor ","metadata":{"execution":{"iopub.status.busy":"2023-01-25T15:23:42.007352Z","iopub.execute_input":"2023-01-25T15:23:42.007731Z","iopub.status.idle":"2023-01-25T15:23:42.016878Z","shell.execute_reply.started":"2023-01-25T15:23:42.007700Z","shell.execute_reply":"2023-01-25T15:23:42.015213Z"}}},{"cell_type":"code","source":"# x_new and y_new we will use for balanced data set for all modules\nrandomsample=  RandomOverSampler()\nx_new, y_new = randomsample.fit_resample(X, y)\n\nfrom collections import Counter\nprint('Original dataset shape  {}'.format(Counter(y)))\nprint('Resampled dataset shape {}'.format(Counter(y_new)))\nsns.countplot(y_new, palette='husl')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:41.224583Z","iopub.execute_input":"2023-01-30T19:05:41.225098Z","iopub.status.idle":"2023-01-30T19:05:42.592212Z","shell.execute_reply.started":"2023-01-30T19:05:41.225057Z","shell.execute_reply":"2023-01-30T19:05:42.591054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_new.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:42.593953Z","iopub.execute_input":"2023-01-30T19:05:42.594328Z","iopub.status.idle":"2023-01-30T19:05:42.633276Z","shell.execute_reply.started":"2023-01-30T19:05:42.594294Z","shell.execute_reply":"2023-01-30T19:05:42.632071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(x_new, y_new, stratify= y_new, test_size= 0.3, random_state= 42)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:42.634640Z","iopub.execute_input":"2023-01-30T19:05:42.635030Z","iopub.status.idle":"2023-01-30T19:05:43.551435Z","shell.execute_reply.started":"2023-01-30T19:05:42.634996Z","shell.execute_reply":"2023-01-30T19:05:43.550223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = MinMaxScaler()\nX_train[X_train.columns] = scaler.fit_transform(X_train[X_train.columns])\nX_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:43.552970Z","iopub.execute_input":"2023-01-30T19:05:43.553342Z","iopub.status.idle":"2023-01-30T19:05:44.015259Z","shell.execute_reply.started":"2023-01-30T19:05:43.553309Z","shell.execute_reply":"2023-01-30T19:05:44.014297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LogisticRegression()\nmodel.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:44.016378Z","iopub.execute_input":"2023-01-30T19:05:44.017297Z","iopub.status.idle":"2023-01-30T19:05:55.908390Z","shell.execute_reply.started":"2023-01-30T19:05:44.017259Z","shell.execute_reply":"2023-01-30T19:05:55.906767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test[X_test.columns] = scaler.transform(X_test[X_test.columns])\ny_pred = model.predict(X_test)\naccuracy, cnf_matrix = model_eval(model, X_test, y_test)\nprint(accuracy)\nprint(cnf_matrix)\ntemp = store_results(\"logistic regres - balance predict\", y_train, X_train, y_test, X_test, model, 5)\ndraw_roc(model, X_test, y_test)\noutcome = outcome.append(temp)\noutcome","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:05:55.916865Z","iopub.execute_input":"2023-01-30T19:05:55.922034Z","iopub.status.idle":"2023-01-30T19:06:13.710771Z","shell.execute_reply.started":"2023-01-30T19:05:55.921944Z","shell.execute_reply":"2023-01-30T19:06:13.709493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Decision tree classifier with unbalanced data ","metadata":{}},{"cell_type":"code","source":"# decision tree doesn't require feature scaling, will use the raw features directly \nX_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size= 0.3, random_state= 42)\nX_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:06:13.712828Z","iopub.execute_input":"2023-01-30T19:06:13.713695Z","iopub.status.idle":"2023-01-30T19:06:14.160185Z","shell.execute_reply.started":"2023-01-30T19:06:13.713649Z","shell.execute_reply":"2023-01-30T19:06:14.158948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_basic = DecisionTreeClassifier(random_state=42)\ndt_basic.fit(X_train, y_train)\ny_preds = dt_basic.predict(X_test)\naccuracy, cnf_matrix = model_eval(dt_basic, X_test, y_test)\nprint(accuracy)\nprint(cnf_matrix)\ntemp = store_results(\"decision tree basic - imbalance predict\", y_train, X_train, y_test, X_test, dt_basic, 5)\ndraw_roc(dt_basic, X_test, y_test)\noutcome = outcome.append(temp)\noutcome\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:06:14.161782Z","iopub.execute_input":"2023-01-30T19:06:14.162632Z","iopub.status.idle":"2023-01-30T19:06:32.692294Z","shell.execute_reply.started":"2023-01-30T19:06:14.162585Z","shell.execute_reply":"2023-01-30T19:06:32.691198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Decision tree classifier with balanced data \n","metadata":{}},{"cell_type":"code","source":"# x_new and y_new are already calculated before for balanced dataset using the same.\n\nX_train, X_test, y_train, y_test = train_test_split(x_new, y_new, stratify= y_new, test_size= 0.3, random_state= 42)\nX_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:06:32.693682Z","iopub.execute_input":"2023-01-30T19:06:32.694023Z","iopub.status.idle":"2023-01-30T19:06:33.455678Z","shell.execute_reply.started":"2023-01-30T19:06:32.693992Z","shell.execute_reply":"2023-01-30T19:06:33.454488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_basic = DecisionTreeClassifier(random_state=42)\ndt_basic.fit(X_train, y_train)\ny_preds = dt_basic.predict(X_test)\naccuracy, cnf_matrix = model_eval(dt_basic, X_test, y_test)\nprint(accuracy)\nprint(cnf_matrix)\ntemp = store_results(\"decision tree basic - balance predict\", y_train, X_train, y_test, X_test, dt_basic, 5)\ndraw_roc(dt_basic, X_test, y_test)\noutcome = outcome.append(temp)\noutcome","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:06:33.456993Z","iopub.execute_input":"2023-01-30T19:06:33.457324Z","iopub.status.idle":"2023-01-30T19:07:05.913710Z","shell.execute_reply.started":"2023-01-30T19:06:33.457296Z","shell.execute_reply":"2023-01-30T19:07:05.912455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### observation \n1. decision tree on unbalanced data isn't doing that great.\n1. decision tree on balanced data is really doing very good. \n2. decision tree on balanced data on test data as well as cross validation is doing really well.\n4. ROC_AUC curve looks really good.","metadata":{}},{"cell_type":"markdown","source":"## Random forest classifier with imbalanced data \n","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size= 0.3, random_state= 42)\nX_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:07:05.915437Z","iopub.execute_input":"2023-01-30T19:07:05.916081Z","iopub.status.idle":"2023-01-30T19:07:06.401903Z","shell.execute_reply.started":"2023-01-30T19:07:05.916038Z","shell.execute_reply":"2023-01-30T19:07:06.400808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_rf = RandomForestClassifier(random_state=42)\nmodel_rf.fit(X_train, y_train)\ny_preds = model_rf.predict(X_test)\naccuracy, cnf_matrix = model_eval(model_rf, X_test, y_test)\nprint(accuracy)\nprint(cnf_matrix)\ntemp = store_results(\"Random forest basic - imbalance predict\", y_train, X_train, y_test, X_test, model_rf, 5)\ndraw_roc(model_rf, X_test, y_test)\noutcome = outcome.append(temp)\noutcome\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:07:06.403234Z","iopub.execute_input":"2023-01-30T19:07:06.403563Z","iopub.status.idle":"2023-01-30T19:12:34.327902Z","shell.execute_reply.started":"2023-01-30T19:07:06.403533Z","shell.execute_reply":"2023-01-30T19:12:34.326685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random forest classifier with balanced data \n","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(x_new, y_new, stratify= y_new, test_size= 0.3, random_state= 42)\nX_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:12:34.329371Z","iopub.execute_input":"2023-01-30T19:12:34.330384Z","iopub.status.idle":"2023-01-30T19:12:35.065143Z","shell.execute_reply.started":"2023-01-30T19:12:34.330348Z","shell.execute_reply":"2023-01-30T19:12:35.064139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_rf = RandomForestClassifier(random_state=42)\nmodel_rf.fit(X_train, y_train)\ny_preds = model_rf.predict(X_test)\naccuracy, cnf_matrix = model_eval(model_rf, X_test, y_test)\nprint(accuracy)\nprint(cnf_matrix)\ntemp = store_results(\"Random forest basic - balance predict\", y_train, X_train, y_test, X_test, model_rf, 5)\ndraw_roc(model_rf, X_test, y_test)\noutcome = outcome.append(temp)\noutcome","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:12:35.066923Z","iopub.execute_input":"2023-01-30T19:12:35.067526Z","iopub.status.idle":"2023-01-30T19:23:45.225314Z","shell.execute_reply.started":"2023-01-30T19:12:35.067494Z","shell.execute_reply":"2023-01-30T19:23:45.223913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### observarion :\n 1. Random forest data with imbalance data is over fitted for training data.\n 1. Random forest with balanced data set is performing the best so far on both train and test data.\n 2. Random forest with balanced data holding good for cross validation as well.\n Conclusion based on simple models built  :- Random forest on balanced data set is the best model built\n","metadata":{}},{"cell_type":"markdown","source":"# Building basic models [with feature engineering]\n in this will follow on building same models as before.\n 1. Logistic regression.\n 2. Decision tree classifier.\n 3. Random forest Classifier.\n One Important observation : While building, we observed that building with balanced predictor is a better apporach. hence all the 3 models in feature engineering case will be built using balanced data set only. ","metadata":{}},{"cell_type":"code","source":"data_feature = sampled_data.copy()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:23:45.227254Z","iopub.execute_input":"2023-01-30T19:23:45.227715Z","iopub.status.idle":"2023-01-30T19:23:45.430450Z","shell.execute_reply.started":"2023-01-30T19:23:45.227665Z","shell.execute_reply":"2023-01-30T19:23:45.429571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = data_feature.corr()\nf, ax = plt.subplots(figsize=(12,12))\nsns.heatmap(corr, ax=ax, annot=True, linewidths=5,cmap='YlGn')\nplt.title(\"Pearson correlation of Features\", y=1.05, size=15)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:23:45.432051Z","iopub.execute_input":"2023-01-30T19:23:45.432772Z","iopub.status.idle":"2023-01-30T19:23:47.281842Z","shell.execute_reply.started":"2023-01-30T19:23:45.432708Z","shell.execute_reply":"2023-01-30T19:23:47.280909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# C1 and device_type are highly corelated, since C1 is anomalised column we can drop it.\n# C14 and C17 are highly co-related, dropping either is a good idea.\n\ndata_feature.drop(['C14', 'C1'], axis=1, inplace= True)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:23:47.283540Z","iopub.execute_input":"2023-01-30T19:23:47.283910Z","iopub.status.idle":"2023-01-30T19:23:47.423271Z","shell.execute_reply.started":"2023-01-30T19:23:47.283877Z","shell.execute_reply":"2023-01-30T19:23:47.422153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = data_feature.drop(['y'], axis=1)\ny = data_feature['y']","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:23:47.424534Z","iopub.execute_input":"2023-01-30T19:23:47.424877Z","iopub.status.idle":"2023-01-30T19:23:47.522242Z","shell.execute_reply.started":"2023-01-30T19:23:47.424846Z","shell.execute_reply":"2023-01-30T19:23:47.521198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_encoder = ce.TargetEncoder()\nX = target_encoder.fit_transform(X, y)\nX.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:23:47.523764Z","iopub.execute_input":"2023-01-30T19:23:47.524220Z","iopub.status.idle":"2023-01-30T19:24:00.354097Z","shell.execute_reply.started":"2023-01-30T19:23:47.524166Z","shell.execute_reply":"2023-01-30T19:24:00.352882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# As discussed at the start, we will use balanced data for all the \n# x_new and y_new we will use for balanced data set for all modules\nrandomsample=  RandomOverSampler()\nx_new, y_new = randomsample.fit_resample(X, y)\n\nfrom collections import Counter\nprint('Original dataset shape  {}'.format(Counter(y)))\nprint('Resampled dataset shape {}'.format(Counter(y_new)))\nsns.countplot(y_new, palette='husl')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:00.355454Z","iopub.execute_input":"2023-01-30T19:24:00.356289Z","iopub.status.idle":"2023-01-30T19:24:01.600407Z","shell.execute_reply.started":"2023-01-30T19:24:00.356241Z","shell.execute_reply":"2023-01-30T19:24:01.599105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(x_new, y_new, stratify= y_new, test_size= 0.3, random_state= 42)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:01.602310Z","iopub.execute_input":"2023-01-30T19:24:01.602817Z","iopub.status.idle":"2023-01-30T19:24:02.435836Z","shell.execute_reply.started":"2023-01-30T19:24:01.602770Z","shell.execute_reply":"2023-01-30T19:24:02.434791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Logistic regression model with feature engineering and balance precidtor.","metadata":{}},{"cell_type":"code","source":"scaler = MinMaxScaler()\nX_train[X_train.columns] = scaler.fit_transform(X_train[X_train.columns])\nX_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:02.437724Z","iopub.execute_input":"2023-01-30T19:24:02.438629Z","iopub.status.idle":"2023-01-30T19:24:02.824655Z","shell.execute_reply.started":"2023-01-30T19:24:02.438581Z","shell.execute_reply":"2023-01-30T19:24:02.823377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_sm = sm.add_constant(X_train)\nlogm2 = sm.GLM(y_train, X_train_sm, family = sm.families.Binomial())\nres = logm2.fit()\nres.summary()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:02.826085Z","iopub.execute_input":"2023-01-30T19:24:02.826445Z","iopub.status.idle":"2023-01-30T19:24:08.524445Z","shell.execute_reply.started":"2023-01-30T19:24:02.826411Z","shell.execute_reply":"2023-01-30T19:24:08.523371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred = res.predict(X_train_sm)\ny_train_pred_final = pd.DataFrame({'y':y_train, 'y_Prob':y_train_pred})\ny_train_pred_final['predicted'] = y_train_pred_final.y_Prob.map(lambda x: 1 if x > 0.5 else 0)\nconfusion = metrics.confusion_matrix(y_train_pred_final.y, y_train_pred_final.predicted )\nprint(confusion)\nprint(metrics.accuracy_score(y_train_pred_final.y, y_train_pred_final.predicted))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:08.525750Z","iopub.execute_input":"2023-01-30T19:24:08.526083Z","iopub.status.idle":"2023-01-30T19:24:09.145656Z","shell.execute_reply.started":"2023-01-30T19:24:08.526053Z","shell.execute_reply":"2023-01-30T19:24:09.144420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sm_model_evaluation (model, x_test, y_test):\n    '''\n    model = sm model\n    y_test = series of labels \n    columns = list of columns in features\n    x_test = test dataframe \n    '''\n    X_sm = sm.add_constant(x_test)\n    y_pred = res.predict(X_sm)\n    y_train_pred_final = pd.DataFrame({'y':y_test, 'y_Prob':y_pred})\n    y_train_pred_final['predicted'] = y_train_pred_final.y_Prob.map(lambda x: 1 if x > 0.5 else 0)\n    # Let's check the overall accuracy.\n    print(metrics.accuracy_score(y_train_pred_final.y, y_train_pred_final.predicted))\n    confusion = metrics.confusion_matrix(y_train_pred_final.y, y_train_pred_final.predicted )\n    print(confusion)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:09.148342Z","iopub.execute_input":"2023-01-30T19:24:09.148908Z","iopub.status.idle":"2023-01-30T19:24:09.158344Z","shell.execute_reply.started":"2023-01-30T19:24:09.148861Z","shell.execute_reply":"2023-01-30T19:24:09.156616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test[X_test.columns] = scaler.transform(X_test[X_test.columns])\nsm_model_evaluation(res, X_test, y_test)  ","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:09.160016Z","iopub.execute_input":"2023-01-30T19:24:09.160818Z","iopub.status.idle":"2023-01-30T19:24:09.634212Z","shell.execute_reply.started":"2023-01-30T19:24:09.160776Z","shell.execute_reply":"2023-01-30T19:24:09.633142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vif = pd.DataFrame()\nvif['Features'] = X_train_sm.columns\nvif['VIF'] = [variance_inflation_factor(X_train_sm.values, i) for i in range(X_train_sm.shape[1])]\nvif['VIF'] = round(vif['VIF'], 2)\nvif = vif.sort_values(by = \"VIF\", ascending = False)\nvif","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:09.635767Z","iopub.execute_input":"2023-01-30T19:24:09.636216Z","iopub.status.idle":"2023-01-30T19:24:31.406895Z","shell.execute_reply.started":"2023-01-30T19:24:09.636172Z","shell.execute_reply":"2023-01-30T19:24:31.405154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# VIF looks good, w.r.t P values hour and dayofweek have got highest, will remove and build the model.\n\nX_train_sm.drop(['hour','day'], axis=1, inplace= True)\nlogm2 = sm.GLM(y_train, X_train_sm, family = sm.families.Binomial())\nres = logm2.fit()\nres.summary()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:31.414391Z","iopub.execute_input":"2023-01-30T19:24:31.419241Z","iopub.status.idle":"2023-01-30T19:24:36.467936Z","shell.execute_reply.started":"2023-01-30T19:24:31.419156Z","shell.execute_reply":"2023-01-30T19:24:36.467020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vif = pd.DataFrame()\nvif['Features'] = X_train_sm.columns\nvif['VIF'] = [variance_inflation_factor(X_train_sm.values, i) for i in range(X_train_sm.shape[1])]\nvif['VIF'] = round(vif['VIF'], 2)\nvif = vif.sort_values(by = \"VIF\", ascending = False)\nvif","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:36.469149Z","iopub.execute_input":"2023-01-30T19:24:36.469451Z","iopub.status.idle":"2023-01-30T19:24:54.304270Z","shell.execute_reply.started":"2023-01-30T19:24:36.469424Z","shell.execute_reply":"2023-01-30T19:24:54.302707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_sm.drop(['dayofweek','device_ip', 'C15'], axis=1, inplace= True)\nlogm2 = sm.GLM(y_train, X_train_sm, family = sm.families.Binomial())\nres = logm2.fit()\nres.summary()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:54.306748Z","iopub.execute_input":"2023-01-30T19:24:54.307696Z","iopub.status.idle":"2023-01-30T19:24:58.776340Z","shell.execute_reply.started":"2023-01-30T19:24:54.307618Z","shell.execute_reply":"2023-01-30T19:24:58.775244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vif = pd.DataFrame()\nvif['Features'] = X_train_sm.columns\nvif['VIF'] = [variance_inflation_factor(X_train_sm.values, i) for i in range(X_train_sm.shape[1])]\nvif['VIF'] = round(vif['VIF'], 2)\nvif = vif.sort_values(by = \"VIF\", ascending = False)\nvif","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:58.777960Z","iopub.execute_input":"2023-01-30T19:24:58.778663Z","iopub.status.idle":"2023-01-30T19:25:12.102982Z","shell.execute_reply.started":"2023-01-30T19:24:58.778618Z","shell.execute_reply":"2023-01-30T19:25:12.100464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model P values and VIF looks good, will find the best threshold for classification.","metadata":{}},{"cell_type":"code","source":"y_train_pred = res.predict(X_train_sm)\ny_train_pred_final = pd.DataFrame({'y':y_train, 'y_Prob':y_train_pred})\ny_train_pred_final['predicted'] = y_train_pred_final.y_Prob.map(lambda x: 1 if x > 0.5 else 0)\nconfusion = metrics.confusion_matrix(y_train_pred_final.y, y_train_pred_final.predicted )\nprint(confusion)\nprint(metrics.accuracy_score(y_train_pred_final.y, y_train_pred_final.predicted))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:12.112749Z","iopub.execute_input":"2023-01-30T19:25:12.114934Z","iopub.status.idle":"2023-01-30T19:25:12.800674Z","shell.execute_reply.started":"2023-01-30T19:25:12.114847Z","shell.execute_reply":"2023-01-30T19:25:12.799159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = list(X_train_sm.columns)\ncol.remove('const')\nsm_model_evaluation(res, X_test[col], y_test)  ","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:12.802195Z","iopub.execute_input":"2023-01-30T19:25:12.802554Z","iopub.status.idle":"2023-01-30T19:25:13.199435Z","shell.execute_reply.started":"2023-01-30T19:25:12.802523Z","shell.execute_reply":"2023-01-30T19:25:13.198017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numbers = [float(x)/20 for x in range(20)]\nprint(numbers)\nfor i in numbers:\n    y_train_pred_final[i]= y_train_pred_final.y_Prob.map(lambda x: 1 if x > i else 0)\ny_train_pred_final.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:13.201158Z","iopub.execute_input":"2023-01-30T19:25:13.201529Z","iopub.status.idle":"2023-01-30T19:25:20.385517Z","shell.execute_reply.started":"2023-01-30T19:25:13.201496Z","shell.execute_reply":"2023-01-30T19:25:20.384235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cutoff_df = pd.DataFrame( columns = ['Thresold_prob','accuracy','recall','precision'])\nfrom sklearn.metrics import confusion_matrix\n\nnum = [0.0,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,0.7,0.75,0.8,0.85,0.9,0.95]\nfor i in num:\n    cm1 = metrics.confusion_matrix(y_train_pred_final.y, y_train_pred_final[i] )\n    total1=sum(sum(cm1))\n    TP = cm1[1,1] # true positive \n    TN = cm1[0,0] # true negatives\n    FP = cm1[0,1] # false positives\n    FN = cm1[1,0] # false negatives\n    accuracy = (TP + TN)/total1\n    \n    recall = TP / (TP+FP)\n    precision = TP / (TP+FN)\n    cutoff_df.loc[i] =[ i ,accuracy,recall,precision]\ncutoff_df","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:20.386979Z","iopub.execute_input":"2023-01-30T19:25:20.387479Z","iopub.status.idle":"2023-01-30T19:25:22.966217Z","shell.execute_reply.started":"2023-01-30T19:25:20.387436Z","shell.execute_reply":"2023-01-30T19:25:22.964864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cutoff_df.plot.line(x='Thresold_prob', y=['accuracy','recall','precision'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:22.967409Z","iopub.execute_input":"2023-01-30T19:25:22.967755Z","iopub.status.idle":"2023-01-30T19:25:23.255483Z","shell.execute_reply.started":"2023-01-30T19:25:22.967712Z","shell.execute_reply":"2023-01-30T19:25:23.254649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred_final['final_predicted'] = y_train_pred_final.y_Prob.map( lambda x: 1 if x > 0.50 else 0)\ny_train_pred_final.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:23.256572Z","iopub.execute_input":"2023-01-30T19:25:23.257425Z","iopub.status.idle":"2023-01-30T19:25:23.629679Z","shell.execute_reply.started":"2023-01-30T19:25:23.257389Z","shell.execute_reply":"2023-01-30T19:25:23.628501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_sm.drop('const', axis= 1, inplace= True)\nX_test = X_test[X_train_sm.columns]\nprint(X_train_sm.shape)\nprint(X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:23.631626Z","iopub.execute_input":"2023-01-30T19:25:23.632137Z","iopub.status.idle":"2023-01-30T19:25:23.661861Z","shell.execute_reply.started":"2023-01-30T19:25:23.632094Z","shell.execute_reply":"2023-01-30T19:25:23.660456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LogisticRegression()\nmodel.fit(X_train_sm, y_train)\naccuracy, cnf_matrix = model_eval(model, X_test[X_train_sm.columns], y_test)\ntemp = store_results(\"logistic reg with feature engine\", y_train, X_train_sm, y_test, X_test[X_train_sm.columns], \\\n                     model, 5)\ndraw_roc(model, X_test[X_train_sm.columns], y_test)\noutcome = outcome.append(temp)\noutcome\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:23.676565Z","iopub.execute_input":"2023-01-30T19:25:23.677004Z","iopub.status.idle":"2023-01-30T19:25:45.378945Z","shell.execute_reply.started":"2023-01-30T19:25:23.676966Z","shell.execute_reply":"2023-01-30T19:25:45.377685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decision tree model with feature engineering and balance precidtor.\n","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(x_new, y_new, stratify= y_new, test_size= 0.3, random_state= 42)\nX_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:45.381032Z","iopub.execute_input":"2023-01-30T19:25:45.382371Z","iopub.status.idle":"2023-01-30T19:25:46.224912Z","shell.execute_reply.started":"2023-01-30T19:25:45.382316Z","shell.execute_reply":"2023-01-30T19:25:46.223793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_tree = DecisionTreeClassifier(random_state=42)\ndt_tree.fit(X_train, y_train)\nfeature_importances = dt_tree.feature_importances_\nfeatures = X_train.columns\ndf = pd.DataFrame({'features': features, 'importance': feature_importances})\ndf.sort_values(by='importance', ascending = False)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:46.226385Z","iopub.execute_input":"2023-01-30T19:25:46.226722Z","iopub.status.idle":"2023-01-30T19:25:56.816922Z","shell.execute_reply.started":"2023-01-30T19:25:46.226690Z","shell.execute_reply":"2023-01-30T19:25:56.815647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df.importance > 0.02]\nrf_cols = []\nfor col in list(X_train.columns):\n    if col in list(df.features):\n        rf_cols.append(col)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:56.818385Z","iopub.execute_input":"2023-01-30T19:25:56.819178Z","iopub.status.idle":"2023-01-30T19:25:56.826470Z","shell.execute_reply.started":"2023-01-30T19:25:56.819141Z","shell.execute_reply":"2023-01-30T19:25:56.825234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_tree = DecisionTreeClassifier(random_state=42)\ndt_tree.fit(X_train[rf_cols], y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:56.828093Z","iopub.execute_input":"2023-01-30T19:25:56.829300Z","iopub.status.idle":"2023-01-30T19:26:03.856884Z","shell.execute_reply.started":"2023-01-30T19:25:56.829253Z","shell.execute_reply":"2023-01-30T19:26:03.855643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(rf_cols)\nX_train = X_train[rf_cols]\nX_test = X_test[rf_cols]\nprint(len(rf_cols))\npredict_rf = dt_tree.predict(X_train)\npredict_rf_test = dt_tree.predict(X_test)\n\naccuracy, cnf_matrix = model_eval(dt_tree, X_train, y_train)\nprint(\"Train results\")\nprint(\"accuracy\",accuracy)\nprint(\"cnf_matrix \\n\",cnf_matrix)\n\naccuracy, cnf_matrix = model_eval(dt_tree, X_test, y_test)\nprint(\"Test results\")\nprint(\"accuracy\",accuracy)\nprint(\"cnf_matrix \\n\",cnf_matrix)\n\ndraw_roc(dt_tree, X_test, y_test)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:26:03.858197Z","iopub.execute_input":"2023-01-30T19:26:03.858542Z","iopub.status.idle":"2023-01-30T19:26:05.476014Z","shell.execute_reply.started":"2023-01-30T19:26:03.858511Z","shell.execute_reply":"2023-01-30T19:26:05.474822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" Above basic tree with right features seems data is overfitting. having the correct hyper parameter tuning help in interpretation and bit of over fitting of the model.","metadata":{}},{"cell_type":"code","source":"param_grid = {\n    'max_depth': range(1,15),\n    'min_samples_leaf': range(10,200,20),\n    'min_samples_split': range(50, 150, 50)\n    \n}\nn_folds = 5\ndtree = DecisionTreeClassifier(random_state= 42)\ntree3 = GridSearchCV(dtree, param_grid, cv=n_folds, n_jobs =-1,return_train_score=True)\ntree3.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:26:05.477512Z","iopub.execute_input":"2023-01-30T19:26:05.477856Z","iopub.status.idle":"2023-01-30T19:50:47.788223Z","shell.execute_reply.started":"2023-01-30T19:26:05.477826Z","shell.execute_reply":"2023-01-30T19:50:47.786419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree3.best_params_","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:50:47.790572Z","iopub.execute_input":"2023-01-30T19:50:47.792008Z","iopub.status.idle":"2023-01-30T19:50:47.801611Z","shell.execute_reply.started":"2023-01-30T19:50:47.791961Z","shell.execute_reply":"2023-01-30T19:50:47.800792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree3.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:50:47.803368Z","iopub.execute_input":"2023-01-30T19:50:47.804291Z","iopub.status.idle":"2023-01-30T19:50:47.821145Z","shell.execute_reply.started":"2023-01-30T19:50:47.804245Z","shell.execute_reply":"2023-01-30T19:50:47.819685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_tree =  DecisionTreeClassifier(max_depth=14, min_samples_leaf=10, min_samples_split=50,\n                       random_state=42)\ndt_tree.fit(X_train, y_train)\naccuracy, cnf_matrix = model_eval(dt_tree, X_test, y_test)\ntemp = store_results(\"Decision tree with feature engine\", y_train, X_train, y_test, X_test, \\\n                     dt_tree, 5)\ndraw_roc(dt_tree, X_test, y_test)\noutcome = outcome.append(temp)\noutcome","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:50:47.823360Z","iopub.execute_input":"2023-01-30T19:50:47.824799Z","iopub.status.idle":"2023-01-30T19:51:03.008746Z","shell.execute_reply.started":"2023-01-30T19:50:47.824715Z","shell.execute_reply":"2023-01-30T19:51:03.006165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random forest model with feature engineering and balance precidtor.","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(x_new, y_new, stratify= y_new, test_size= 0.3, random_state= 42)\nX_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:51:03.010498Z","iopub.execute_input":"2023-01-30T19:51:03.011376Z","iopub.status.idle":"2023-01-30T19:51:03.776847Z","shell.execute_reply.started":"2023-01-30T19:51:03.011327Z","shell.execute_reply":"2023-01-30T19:51:03.775997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_rf = RandomForestClassifier(random_state=42)\nmodel_rf.fit(X_train, y_train)\nfeature_importances = model_rf.feature_importances_\nfeatures = X_train.columns\ndf = pd.DataFrame({'features': features, 'importance': feature_importances})\ndf.sort_values(by='importance', ascending = False)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:51:03.778237Z","iopub.execute_input":"2023-01-30T19:51:03.778545Z","iopub.status.idle":"2023-01-30T19:54:03.729459Z","shell.execute_reply.started":"2023-01-30T19:51:03.778518Z","shell.execute_reply":"2023-01-30T19:54:03.728268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"using random forest feature importance metric to decide on best features and building the model.","metadata":{}},{"cell_type":"code","source":"df = df[df.importance > 0.02]\nrf_cols = []\nfor col in list(X_train.columns):\n    if col in list(df.features):\n        rf_cols.append(col)\n        \nX_train = X_train[rf_cols]\nX_test = X_test[rf_cols]\n\nmodel_rf = RandomForestClassifier(random_state=42)\nmodel_rf.fit(X_train, y_train)\nprint(rf_cols)\n\nprint(len(rf_cols))\npredict_rf = model_rf.predict(X_train)\npredict_rf_test = model_rf.predict(X_test)\n\naccuracy, cnf_matrix = model_eval(model_rf, X_train, y_train)\nprint(\"Train results\")\nprint(\"accuracy\",accuracy)\nprint(\"cnf_matrix \\n\", cnf_matrix)\n\naccuracy, cnf_matrix = model_eval(model_rf, X_test, y_test)\nprint(\"Test results\")\nprint(\"accuracy\",accuracy)\nprint(\"cnf_matrix \\n\", cnf_matrix)\n\ndraw_roc(model_rf, X_test, y_test)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:54:03.731230Z","iopub.execute_input":"2023-01-30T19:54:03.732353Z","iopub.status.idle":"2023-01-30T19:59:45.124451Z","shell.execute_reply.started":"2023-01-30T19:54:03.732314Z","shell.execute_reply":"2023-01-30T19:59:45.123103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = store_results(\"Random Forest with feature engine\", y_train, X_train, y_test, X_test, \\\n                     model_rf, 5)\noutcome = outcome.append(temp)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:59:45.126990Z","iopub.execute_input":"2023-01-30T19:59:45.127946Z","iopub.status.idle":"2023-01-30T20:07:36.852522Z","shell.execute_reply.started":"2023-01-30T19:59:45.127890Z","shell.execute_reply":"2023-01-30T20:07:36.850713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outcome.reset_index(drop=True, inplace=True)\noutcome","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:07:36.855151Z","iopub.execute_input":"2023-01-30T20:07:36.855653Z","iopub.status.idle":"2023-01-30T20:07:36.890590Z","shell.execute_reply.started":"2023-01-30T20:07:36.855601Z","shell.execute_reply":"2023-01-30T20:07:36.889365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# conclusion\n If we have to select one model, Random forest classifier with feature enginnering looks promising and best. although after the feature engineering the training and test results looks same as before feature engineering, but model is very robust with new features and rightly fitted for both training and test dataset. \n Decisiontree  and logistic regression classifer seems to have low accuracy, precision, and recall, overall random forest classifier seems doing better with all aspects.\n","metadata":{}},{"cell_type":"code","source":"plt.rcParams[\"figure.figsize\"] = (10,8)\noutcome.plot(x='Model', y=['Accuracy_train','Accuracy_test','CrossVal_Mean'], kind=\"bar\")\nplt.xticks(rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:07:36.892225Z","iopub.execute_input":"2023-01-30T20:07:36.892582Z","iopub.status.idle":"2023-01-30T20:07:37.291488Z","shell.execute_reply.started":"2023-01-30T20:07:36.892551Z","shell.execute_reply":"2023-01-30T20:07:37.290677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing test data for sumbission","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('../input/avazu-ctr-prediction/test.gz', parse_dates=['hour'], date_parser=parse_date)\ntest.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:07:37.292958Z","iopub.execute_input":"2023-01-30T20:07:37.293303Z","iopub.status.idle":"2023-01-30T20:09:17.594822Z","shell.execute_reply.started":"2023-01-30T20:07:37.293265Z","shell.execute_reply":"2023-01-30T20:09:17.593835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['month'] = test['hour'].dt.month\ntest['dayofweek'] = test['hour'].dt.dayofweek\ntest['day'] = test['hour'].dt.day\ntest['hour_time'] = test['hour'].dt.hour\ntest.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:17.596460Z","iopub.execute_input":"2023-01-30T20:09:17.597854Z","iopub.status.idle":"2023-01-30T20:09:19.397939Z","shell.execute_reply.started":"2023-01-30T20:09:17.597803Z","shell.execute_reply":"2023-01-30T20:09:19.396661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = ['C15', 'C16', 'C19', 'C21']\nfor col in col:\n    percentiles = test[col].quantile(0.98)\n    if test[col].quantile(0.98) < 0.5 * test[col].max():\n        test[col][test[col] >= percentiles] = percentiles\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:19.399383Z","iopub.execute_input":"2023-01-30T20:09:19.400537Z","iopub.status.idle":"2023-01-30T20:09:19.935230Z","shell.execute_reply.started":"2023-01-30T20:09:19.400501Z","shell.execute_reply":"2023-01-30T20:09:19.934024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntest.drop(['month', 'C20'], axis=1, inplace=True)\nlabelEncoder= LabelEncoder()\nfor col in (test.columns):\n    if test[col].dtype == \"object\":\n        test[col] = labelEncoder.fit_transform(test[col])\n\ntest.head(2)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:19.938783Z","iopub.execute_input":"2023-01-30T20:09:19.939832Z","iopub.status.idle":"2023-01-30T20:09:19.947046Z","shell.execute_reply.started":"2023-01-30T20:09:19.939778Z","shell.execute_reply":"2023-01-30T20:09:19.946034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(['id', 'hour'], axis = 1, inplace = True) \ntest.rename(columns={'hour_time': 'hour'},\n          inplace=True, errors='raise')\ntest.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:19.949004Z","iopub.execute_input":"2023-01-30T20:09:19.949796Z","iopub.status.idle":"2023-01-30T20:09:21.338518Z","shell.execute_reply.started":"2023-01-30T20:09:19.949717Z","shell.execute_reply":"2023-01-30T20:09:21.337424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(['month', 'C20'], axis=1, inplace=True)\ntest.drop(['C14', 'C1'], axis=1, inplace= True)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:21.339870Z","iopub.execute_input":"2023-01-30T20:09:21.340240Z","iopub.status.idle":"2023-01-30T20:09:23.025639Z","shell.execute_reply.started":"2023-01-30T20:09:21.340207Z","shell.execute_reply":"2023-01-30T20:09:23.024312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nfor col in (test.columns):\n    print(col)\n    if test[col].dtype == \"object\":\n        test[col] = target_encoder.transform(test)\n\ntest.head(2)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:23.027128Z","iopub.execute_input":"2023-01-30T20:09:23.027866Z","iopub.status.idle":"2023-01-30T20:09:23.034925Z","shell.execute_reply.started":"2023-01-30T20:09:23.027830Z","shell.execute_reply":"2023-01-30T20:09:23.033646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test =  target_encoder.transform(test)\ntest.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:23.035922Z","iopub.execute_input":"2023-01-30T20:09:23.036237Z","iopub.status.idle":"2023-01-30T20:09:45.999363Z","shell.execute_reply.started":"2023-01-30T20:09:23.036210Z","shell.execute_reply":"2023-01-30T20:09:45.998154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[test.columns]  = scaler.transform(test[test.columns])\ntest.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:46.001421Z","iopub.execute_input":"2023-01-30T20:09:46.002305Z","iopub.status.idle":"2023-01-30T20:09:48.700177Z","shell.execute_reply.started":"2023-01-30T20:09:46.002258Z","shell.execute_reply":"2023-01-30T20:09:48.698930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# random forest with balanced data was our best model, hence using the same for submission.\nrf_cols","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:48.701785Z","iopub.execute_input":"2023-01-30T20:09:48.702236Z","iopub.status.idle":"2023-01-30T20:09:48.709699Z","shell.execute_reply.started":"2023-01-30T20:09:48.702195Z","shell.execute_reply":"2023-01-30T20:09:48.708522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test[rf_cols]\npredict_sub = model_rf.predict(test)\nprint(len(predict_sub))\nprint(print(len(submission)))\nprint(predict_sub)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:09:48.711366Z","iopub.execute_input":"2023-01-30T20:09:48.712121Z","iopub.status.idle":"2023-01-30T20:11:09.261030Z","shell.execute_reply.started":"2023-01-30T20:09:48.712077Z","shell.execute_reply":"2023-01-30T20:11:09.259047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['click'] = predict_sub\nsubmission.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:11:09.265101Z","iopub.execute_input":"2023-01-30T20:11:09.265479Z","iopub.status.idle":"2023-01-30T20:11:09.326842Z","shell.execute_reply.started":"2023-01-30T20:11:09.265445Z","shell.execute_reply":"2023-01-30T20:11:09.324447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T20:11:09.330940Z","iopub.execute_input":"2023-01-30T20:11:09.332070Z","iopub.status.idle":"2023-01-30T20:11:20.198944Z","shell.execute_reply.started":"2023-01-30T20:11:09.331958Z","shell.execute_reply":"2023-01-30T20:11:20.197778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"********************************************************** END **********************************************************","metadata":{}}]}