{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-07T05:44:07.636351Z","iopub.execute_input":"2023-04-07T05:44:07.636837Z","iopub.status.idle":"2023-04-07T05:44:07.646603Z","shell.execute_reply.started":"2023-04-07T05:44:07.636772Z","shell.execute_reply":"2023-04-07T05:44:07.644721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\nsns.set_theme(style=\"darkgrid\")\npd.set_option('display.max_columns', 0)\nplt.style.use('ggplot')\npd.options.display.float_format = '{:.2f}'.format\nimport math\nfrom sklearn.preprocessing import LabelEncoder \nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.dummy import DummyClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom imblearn.over_sampling import RandomOverSampler\nfrom sklearn.tree import DecisionTreeClassifier\nfrom IPython.display import Image  \nfrom six import StringIO\nfrom sklearn.tree import export_graphviz\nimport graphviz\nfrom sklearn.ensemble import RandomForestClassifier\nimport statsmodels.api as sm\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom sklearn.feature_selection import RFE\nfrom sklearn.model_selection import GridSearchCV\nimport random\nimport gzip\nimport category_encoders as ce\npd.options.display.float_format = '{:.2f}'.format","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:44:07.648884Z","iopub.execute_input":"2023-04-07T05:44:07.650042Z","iopub.status.idle":"2023-04-07T05:44:07.664831Z","shell.execute_reply.started":"2023-04-07T05:44:07.649994Z","shell.execute_reply":"2023-04-07T05:44:07.663699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnum_records = 40428967\nsample_size = 5000000\nskip_values = sorted(random.sample(range(1,num_records), num_records - sample_size))\nparse_date = lambda val : pd.datetime.strptime(val, '%y%m%d%H')\ntrain = pd.read_csv( \"../input/avazu-ctr-prediction/train.gz\", parse_dates=['hour'],chunksize=500000, date_parser=parse_date,\\\n                     skiprows=skip_values)\ntrain=next(train)\n# train = pd.read_csv(\"../input/avazu-ctr-prediction/train.gz\", parse_dates=['hour'], date_parser=parse_date,\\\n#                      skiprows=skip_values)\n# train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:44:07.666894Z","iopub.execute_input":"2023-04-07T05:44:07.667256Z","iopub.status.idle":"2023-04-07T05:46:21.117081Z","shell.execute_reply.started":"2023-04-07T05:44:07.667224Z","shell.execute_reply":"2023-04-07T05:46:21.115362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Reading the test data.\n\n# test = pd.read_csv('../input/avazu-ctr-prediction/test.gz', parse_dates=['hour'], date_parser=parse_date)\n# test.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.119601Z","iopub.execute_input":"2023-04-07T05:46:21.120019Z","iopub.status.idle":"2023-04-07T05:46:21.126275Z","shell.execute_reply.started":"2023-04-07T05:46:21.119983Z","shell.execute_reply":"2023-04-07T05:46:21.124347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# submission = pd.read_csv('../input/avazu-ctr-prediction/sampleSubmission.gz')\n# submission.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.131147Z","iopub.execute_input":"2023-04-07T05:46:21.131798Z","iopub.status.idle":"2023-04-07T05:46:21.143517Z","shell.execute_reply.started":"2023-04-07T05:46:21.131735Z","shell.execute_reply":"2023-04-07T05:46:21.141931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train= train.sample(frac = 0.2,random_state=1)\n# test= test.sample(frac = 0.01,random_state=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.145628Z","iopub.execute_input":"2023-04-07T05:46:21.146627Z","iopub.status.idle":"2023-04-07T05:46:21.365014Z","shell.execute_reply.started":"2023-04-07T05:46:21.146563Z","shell.execute_reply":"2023-04-07T05:46:21.363764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datecolumn=['hour']\nfor col in datecolumn:\n    train[col+'month'] = train[col].dt.month\n    train[col+'dayofweek'] = train[col].dt.dayofweek\n    train[col+'day'] = train[col].dt.day\n    train[col+'hour_time'] = train[col].dt.hour\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.366899Z","iopub.execute_input":"2023-04-07T05:46:21.367749Z","iopub.status.idle":"2023-04-07T05:46:21.463747Z","shell.execute_reply.started":"2023-04-07T05:46:21.367695Z","shell.execute_reply":"2023-04-07T05:46:21.462350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncol = ['C15', 'C16', 'C19', 'C21']\nfor col in col:\n    percentiles = train[col].quantile(0.98)\n    if train[col].quantile(0.98) < 0.5 * train[col].max():\n        train[col][train[col] >= percentiles] = percentiles","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.465714Z","iopub.execute_input":"2023-04-07T05:46:21.466565Z","iopub.status.idle":"2023-04-07T05:46:21.497102Z","shell.execute_reply.started":"2023-04-07T05:46:21.466513Z","shell.execute_reply":"2023-04-07T05:46:21.496070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnumerical_features = []\ncategorical_features = []\n\nfor col in (list(train.columns)):\n    if train[col].dtype == \"object\" or train[col].dtype == \"category\" :\n        categorical_features.append(col)\n    else:\n        numerical_features.append(col)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.498850Z","iopub.execute_input":"2023-04-07T05:46:21.499606Z","iopub.status.idle":"2023-04-07T05:46:21.509274Z","shell.execute_reply.started":"2023-04-07T05:46:21.499555Z","shell.execute_reply":"2023-04-07T05:46:21.507775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"numerical columns = \",numerical_features)\nprint(\"\\ncategorical columns = \",categorical_features)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.514949Z","iopub.execute_input":"2023-04-07T05:46:21.515606Z","iopub.status.idle":"2023-04-07T05:46:21.524506Z","shell.execute_reply.started":"2023-04-07T05:46:21.515552Z","shell.execute_reply":"2023-04-07T05:46:21.522674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label='click'","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.526465Z","iopub.execute_input":"2023-04-07T05:46:21.526925Z","iopub.status.idle":"2023-04-07T05:46:21.537548Z","shell.execute_reply.started":"2023-04-07T05:46:21.526886Z","shell.execute_reply":"2023-04-07T05:46:21.535850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[label].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.539458Z","iopub.execute_input":"2023-04-07T05:46:21.540006Z","iopub.status.idle":"2023-04-07T05:46:21.554615Z","shell.execute_reply.started":"2023-04-07T05:46:21.539947Z","shell.execute_reply":"2023-04-07T05:46:21.553068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Drop highly corelated columns","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\n# Create correlation matrix\ncorr_matrix = train.corr().abs()\n\n# Select upper triangle of correlation matrix\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))\n\n# Find features with correlation greater than 0.95\nto_drop = [column for column in upper.columns if any(upper[column] > 0.95)]\n# Drop features \ntrain.drop(to_drop, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.555735Z","iopub.execute_input":"2023-04-07T05:46:21.557009Z","iopub.status.idle":"2023-04-07T05:46:21.711630Z","shell.execute_reply.started":"2023-04-07T05:46:21.556961Z","shell.execute_reply":"2023-04-07T05:46:21.710299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_unique_ids=[]\n\nfor col in (list(train.columns)):\n    if len(list(train[col].unique())) == train.shape[0]:\n        drop_unique_ids.append(col)\ndrop_unique_ids","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.713325Z","iopub.execute_input":"2023-04-07T05:46:21.713736Z","iopub.status.idle":"2023-04-07T05:46:21.904590Z","shell.execute_reply.started":"2023-04-07T05:46:21.713699Z","shell.execute_reply":"2023-04-07T05:46:21.903047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if(len(drop_unique_ids)>0):\n    train.drop(drop_unique_ids, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.906268Z","iopub.execute_input":"2023-04-07T05:46:21.906666Z","iopub.status.idle":"2023-04-07T05:46:21.940072Z","shell.execute_reply.started":"2023-04-07T05:46:21.906620Z","shell.execute_reply":"2023-04-07T05:46:21.938674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if(len(datecolumn)>0):\n    train.drop(datecolumn, axis=1, inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.941630Z","iopub.execute_input":"2023-04-07T05:46:21.942024Z","iopub.status.idle":"2023-04-07T05:46:21.978250Z","shell.execute_reply.started":"2023-04-07T05:46:21.941979Z","shell.execute_reply":"2023-04-07T05:46:21.976820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(1)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:21.980289Z","iopub.execute_input":"2023-04-07T05:46:21.980701Z","iopub.status.idle":"2023-04-07T05:46:22.027627Z","shell.execute_reply.started":"2023-04-07T05:46:21.980663Z","shell.execute_reply":"2023-04-07T05:46:22.025970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique_ids=[]\n\n# for col in (list(train.columns)):\n#     if len(list(train[col].unique())) == train.shape[0]:\n#         drop_unique_ids.append(col)\n# drop_unique_ids","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:22.029488Z","iopub.execute_input":"2023-04-07T05:46:22.029902Z","iopub.status.idle":"2023-04-07T05:46:22.229291Z","shell.execute_reply.started":"2023-04-07T05:46:22.029866Z","shell.execute_reply":"2023-04-07T05:46:22.227325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if(len(drop_unique_ids)>0):\n#     train.drop(drop_unique_ids, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:22.231618Z","iopub.execute_input":"2023-04-07T05:46:22.232132Z","iopub.status.idle":"2023-04-07T05:46:22.240018Z","shell.execute_reply.started":"2023-04-07T05:46:22.232089Z","shell.execute_reply":"2023-04-07T05:46:22.237909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:22.242693Z","iopub.execute_input":"2023-04-07T05:46:22.243170Z","iopub.status.idle":"2023-04-07T05:46:22.294450Z","shell.execute_reply.started":"2023-04-07T05:46:22.243130Z","shell.execute_reply":"2023-04-07T05:46:22.292647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"sample data","metadata":{}},{"cell_type":"code","source":"X = train.drop([label], axis=1)\ny = train[label]","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:22.297091Z","iopub.execute_input":"2023-04-07T05:46:22.297542Z","iopub.status.idle":"2023-04-07T05:46:22.327891Z","shell.execute_reply.started":"2023-04-07T05:46:22.297503Z","shell.execute_reply":"2023-04-07T05:46:22.326379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nfor c in categorical_features:\n    le = LabelEncoder()\n    le.fit(X[c])\n    X[c] = le.transform(X[c])","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:22.333495Z","iopub.execute_input":"2023-04-07T05:46:22.333999Z","iopub.status.idle":"2023-04-07T05:46:23.153605Z","shell.execute_reply.started":"2023-04-07T05:46:22.333954Z","shell.execute_reply":"2023-04-07T05:46:23.152514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_features.remove('id')\nnumerical_features.remove('click')\nnumerical_features.remove('hour')\nnumerical_features.remove('C17')\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:23.155379Z","iopub.execute_input":"2023-04-07T05:46:23.156107Z","iopub.status.idle":"2023-04-07T05:46:23.162982Z","shell.execute_reply.started":"2023-04-07T05:46:23.156064Z","shell.execute_reply":"2023-04-07T05:46:23.160724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nscaler = MinMaxScaler()\nX[numerical_features] = scaler.fit_transform(X[numerical_features])","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:23.165115Z","iopub.execute_input":"2023-04-07T05:46:23.166484Z","iopub.status.idle":"2023-04-07T05:46:23.305750Z","shell.execute_reply.started":"2023-04-07T05:46:23.166424Z","shell.execute_reply":"2023-04-07T05:46:23.304066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nprint(train[label].value_counts(normalize = True))\nprint(\"\\n\")\nplt.figure()\nsns.countplot(x=y)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:23.314892Z","iopub.execute_input":"2023-04-07T05:46:23.315344Z","iopub.status.idle":"2023-04-07T05:46:23.550511Z","shell.execute_reply.started":"2023-04-07T05:46:23.315306Z","shell.execute_reply":"2023-04-07T05:46:23.548854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Train Without Handling data Imbalance","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train,x_cv,y_train,y_cv=train_test_split(X,y,test_size=0.2,random_state=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:23.552377Z","iopub.execute_input":"2023-04-07T05:46:23.552750Z","iopub.status.idle":"2023-04-07T05:46:23.597065Z","shell.execute_reply.started":"2023-04-07T05:46:23.552717Z","shell.execute_reply":"2023-04-07T05:46:23.595291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model train\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import roc_auc_score,roc_curve\nimport statsmodels.api as sm\nfrom sklearn.metrics import precision_score,recall_score\n#gives model report in dataframe\ndef model_report(model,training_x,training_y,testing_x,testing_y,name) :\n    model.fit(training_x,training_y)\n    predictions  = model.predict(testing_x)\n    accuracy     = accuracy_score(testing_y,predictions)\n    recallscore  = recall_score(testing_y,predictions)\n    precision    = precision_score(testing_y,predictions)\n    roc_auc      = roc_auc_score(testing_y,predictions)\n    f1score      = f1_score(testing_y,predictions) \n    kappa_metric = cohen_kappa_score(testing_y,predictions)\n    \n    df = pd.DataFrame({\"Model\"           : [name],\n                       \"Accuracy_score\"  : [accuracy],\n                        \"ROC Accuracy \"    : [roc_auc],\n                       \"Recall_score\"    : [recallscore],\n                       \"Precision\"       : [precision],\n                       \"f1_score\"        : [f1score],\n                       \"Area_under_curve\": [roc_auc],\n                       \"Kappa_metric\"    : [kappa_metric],\n                      })\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:23.599361Z","iopub.execute_input":"2023-04-07T05:46:23.599809Z","iopub.status.idle":"2023-04-07T05:46:23.613089Z","shell.execute_reply.started":"2023-04-07T05:46:23.599755Z","shell.execute_reply":"2023-04-07T05:46:23.611901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n###  Logistic regression\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score\n\nlogistic_model = LogisticRegression(random_state=1)\n#outputs for every model\nmodel1 = model_report(logistic_model,x_train,y_train,x_cv,y_cv, \"Logistic Regression\")\n\nmodel1","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:23.614571Z","iopub.execute_input":"2023-04-07T05:46:23.615004Z","iopub.status.idle":"2023-04-07T05:46:25.317277Z","shell.execute_reply.started":"2023-04-07T05:46:23.614966Z","shell.execute_reply":"2023-04-07T05:46:25.315223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n### Decision Tree\n\nfrom sklearn.tree import DecisionTreeClassifier\ntree_model = DecisionTreeClassifier(max_depth = 9,\n                                       random_state = 123,\n                                       splitter  = \"best\",\n                                       criterion = \"gini\",\n                                      )\n#outputs for every model\nmodel2 = model_report(tree_model,x_train,y_train,x_cv,y_cv, \" Decision Tree\")\nmodel2","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:25.320217Z","iopub.execute_input":"2023-04-07T05:46:25.321382Z","iopub.status.idle":"2023-04-07T05:46:25.936757Z","shell.execute_reply.started":"2023-04-07T05:46:25.321310Z","shell.execute_reply":"2023-04-07T05:46:25.935859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n### Random Forest\n\nfrom sklearn.ensemble import RandomForestClassifier\nforest_model = RandomForestClassifier(n_estimators = 1000,\n                             max_depth = 9,\n                             criterion = \"gini\")\n#forest_model.fit(x_train,y_train)\n#outputs for every model\nmodel3 = model_report(forest_model,x_train,y_train,x_cv,y_cv, \"Random Forest\")\nmodel3","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:46:25.938468Z","iopub.execute_input":"2023-04-07T05:46:25.939127Z","iopub.status.idle":"2023-04-07T05:47:58.973037Z","shell.execute_reply.started":"2023-04-07T05:46:25.939088Z","shell.execute_reply":"2023-04-07T05:47:58.971994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #### Grid search\n\n# from sklearn.model_selection import GridSearchCV\n# paramgrid = {'max_depth': list(range(1,20,2)),'n_estimators':list(range(1,200,20))}\n# grid_search = GridSearchCV(RandomForestClassifier(random_state=1),paramgrid)\n\n# grid_search.fit(x_train,y_train)\n\n# grid_search.best_estimator_\n\n# grid_forest_model = grid_search.best_estimator_\n\n# #outputs for every model\n# model4 = model_report(grid_forest_model,x_train,y_train,x_cv,y_cv, \"Random Forest(Parameter Tunning)\")\n# model4\n\n\n# pred_cv_forest=forest_model.predict(x_cv)\n# score_forest = accuracy_score(pred_cv_forest,y_cv)*100\n# score_forest\n\n# importances = pd.Series(forest_model.feature_importances_,index=X.columns)\n# importances.plot(kind='barh', figsize=(12,8))\n\n\n\n# grid_forest_model.fit(x_train,y_train)\n# pred_grid_forest = grid_forest_model.predict(x_cv)\n# score_grid_forest = accuracy_score(pred_grid_forest,y_cv)*100\n# score_grid_forest\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:47:58.974724Z","iopub.execute_input":"2023-04-07T05:47:58.975476Z","iopub.status.idle":"2023-04-07T05:47:58.981030Z","shell.execute_reply.started":"2023-04-07T05:47:58.975434Z","shell.execute_reply":"2023-04-07T05:47:58.979259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n### XGBOOST\n\nfrom xgboost import XGBClassifier\nscale_pos_weight=1\ny1=y_train[y_train==1].count()\ny0=y_train[y_train==0].count()\nif(y1>y0):\n  scale_pos_weight=(y1/y0)\nelse:\n  scale_pos_weight=(y0/y1)\n  \nxgb_model = XGBClassifier( booster='gbtree', \n                 gamma=0, learning_rate=0.1,\n                    max_depth = 30, min_child_weight=1,  \n                    n_jobs=1,  objective='binary:logistic', \n                    reg_alpha=0, reg_lambda=1, scale_pos_weight=scale_pos_weight,n_estimators=2000)\n\n\n# xgb_model.fit(x_train,y_train)\n#outputs for every model\nmodel5 = model_report(xgb_model,x_train,y_train,x_cv,y_cv, \"XGBOOST\")\nmodel5\n\n\n# param_test = {\n#  'max_depth':[4,5,6],\n#  'min_child_weight':[4,5,6],\n#  'n_estimators':[100,150]\n# }\n# gsearch2 = GridSearchCV(estimator = XGBClassifier( learning_rate=0.1, \n#  min_child_weight=2, gamma=0, subsample=0.8, colsample_bytree=0.8,\n#  objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n#  param_grid = param_test, scoring='roc_auc',n_jobs=4, cv=5)\n# gsearch2.fit(x_train,y_train)\n\n\n# gsearch2.best_estimator_\n\n# grid_xgmodel = gsearch2.best_estimator_\n\n# grid_xgmodel.fit(x_train,y_train)\n\n# pred_xgb=grid_xgmodel.predict(x_cv)\n# score_xgb = accuracy_score(pred_xgb,y_cv)*100\n# score_xgb\n\n# grid_xgmodel.save_model(\"model.json\")\n# model_xgb_2 = XGBClassifier()\n# model_xgb_2.load_model(\"model.json\")\n# model_xgb_2.predict(x_test.head(1))\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:15:52.004676Z","iopub.execute_input":"2023-04-07T06:15:52.005136Z","iopub.status.idle":"2023-04-07T06:43:03.499012Z","shell.execute_reply.started":"2023-04-07T06:15:52.005099Z","shell.execute_reply":"2023-04-07T06:43:03.497336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ## SVC\n\n# from sklearn.svm import SVC\n# svc_lin  = SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n#                decision_function_shape='ovr', degree=3, gamma=1.0, kernel='linear',\n#                max_iter=-1, probability=True, random_state=None, shrinking=True,\n#                tol=0.001, verbose=False)\n# svc_rbf  = SVC(C=1.0, kernel='rbf', \n#                degree= 3, gamma=1.0, \n#                coef0=0.0, shrinking=True,\n#                probability=True,tol=0.001,\n#                cache_size=200, class_weight=None,\n#                verbose=False,max_iter= -1,\n#                random_state=None)\n# model8 = model_report(svc_lin,x_train,y_train,x_cv,y_cv,\n#                       \"SVM Classifier Linear\")\n# model9 = model_report(svc_rbf,x_train,y_train,x_cv,y_cv,\n#                       \"SVM Classifier RBF\")\n\n# model8\n\n# model9","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:15:33.774939Z","iopub.status.idle":"2023-04-07T06:15:33.775440Z","shell.execute_reply.started":"2023-04-07T06:15:33.775212Z","shell.execute_reply":"2023-04-07T06:15:33.775237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n## KNN\n\nfrom sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n           weights='uniform')\nmodel6= model_report(knn,x_train,y_train,x_cv,y_cv,\n                      \"KNN Classifier\")\nmodel6\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:43:03.504227Z","iopub.execute_input":"2023-04-07T06:43:03.504688Z","iopub.status.idle":"2023-04-07T06:43:38.764572Z","shell.execute_reply.started":"2023-04-07T06:43:03.504640Z","shell.execute_reply":"2023-04-07T06:43:38.762931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## GaussianNB\n\nfrom sklearn.naive_bayes import GaussianNB\ngnb = GaussianNB(priors=None)\nmodel7 = model_report(gnb,x_train,y_train,x_cv,y_cv,\n                      \"Naive Bayes\")\nmodel7\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:43:38.766488Z","iopub.execute_input":"2023-04-07T06:43:38.766927Z","iopub.status.idle":"2023-04-07T06:43:38.879520Z","shell.execute_reply.started":"2023-04-07T06:43:38.766886Z","shell.execute_reply":"2023-04-07T06:43:38.877905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## LightGBM\n\nfrom lightgbm import LGBMClassifier\nlearning_rate = 0.5\nnum_leaves = 30\nmin_data_in_leaf = 100\nnum_boost_round = 250\nparams = {\"objective\": \"binary\",\n          \"boosting_type\": \"gbdt\",\n          \"learning_rate\": learning_rate,\n          \"num_leaves\": num_leaves,\n          \"verbosity\": 0,\n          \"drop_rate\": 0.1,\n          \"is_unbalance\": True,\n          \"max_drop\": 50,\n          \"min_child_samples\": 100,\n          \"min_child_weight\": 250,\n     \n          \"n_estimators\":2000\n          }\nlgbm_c = LGBMClassifier(**params,force_row_wise=True)\nmodel10 = model_report(lgbm_c,x_train,y_train,x_cv,y_cv,\n                      \"LGBM Classifier\")\n\nmodel10\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:43:38.883499Z","iopub.execute_input":"2023-04-07T06:43:38.884067Z","iopub.status.idle":"2023-04-07T06:44:06.751116Z","shell.execute_reply.started":"2023-04-07T06:43:38.884011Z","shell.execute_reply":"2023-04-07T06:44:06.749855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary\n","metadata":{}},{"cell_type":"code","source":"\n\n#concat all models\nmodel_performances = pd.concat([model1,model2,model3,\n                                model5,model6,\n                                model7,#model8,model9,\n                                model10],axis = 0).reset_index()\n\nmodel_performances = model_performances.drop(columns = \"index\",axis =1)\n\nmodel_performances","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:44:06.752636Z","iopub.execute_input":"2023-04-07T06:44:06.752969Z","iopub.status.idle":"2023-04-07T06:44:06.785157Z","shell.execute_reply.started":"2023-04-07T06:44:06.752938Z","shell.execute_reply":"2023-04-07T06:44:06.783959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SMOTE\n","metadata":{}},{"cell_type":"code","source":"\nfrom imblearn.over_sampling import SMOTE\nsm = SMOTE(random_state=1)\nX_SM, Y_SM = sm.fit_resample(X, y)\n\ny.value_counts().plot(kind='bar', title='Count of Target Classes in Dataset Before Sampling')\n\nY_SM.value_counts().plot(kind='bar', title='Count of Target Classes in Dataset After Sampling')\nX_Test,Y_Test=x_cv,y_cv","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:44:06.786908Z","iopub.execute_input":"2023-04-07T06:44:06.787373Z","iopub.status.idle":"2023-04-07T06:44:13.695616Z","shell.execute_reply.started":"2023-04-07T06:44:06.787326Z","shell.execute_reply":"2023-04-07T06:44:13.693901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n###  Logistic regression \n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score\n\nlogistic_model = LogisticRegression(random_state=1)\n#outputs for every model\nmodel1 = model_report(logistic_model,X_SM,Y_SM,X_Test,Y_Test, \"Logistic Regression\")\n\nmodel1\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:44:13.697342Z","iopub.execute_input":"2023-04-07T06:44:13.697759Z","iopub.status.idle":"2023-04-07T06:44:15.372463Z","shell.execute_reply.started":"2023-04-07T06:44:13.697715Z","shell.execute_reply":"2023-04-07T06:44:15.371156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n### Decision Tree\n\nfrom sklearn.tree import DecisionTreeClassifier\ntree_model = DecisionTreeClassifier(max_depth = 9,\n                                       random_state = 123,\n                                       splitter  = \"best\",\n                                       criterion = \"gini\",\n                                      )\n#outputs for every model\nmodel2 = model_report(tree_model,X_SM,Y_SM,X_Test,Y_Test, \" Decision Tree\")\nmodel2","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:44:15.374384Z","iopub.execute_input":"2023-04-07T06:44:15.375289Z","iopub.status.idle":"2023-04-07T06:44:17.165294Z","shell.execute_reply.started":"2023-04-07T06:44:15.375239Z","shell.execute_reply":"2023-04-07T06:44:17.163899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n### Random Forest\n\nfrom sklearn.ensemble import RandomForestClassifier\nforest_model = RandomForestClassifier(n_estimators = 1000,\n                             max_depth = 9,\n                             criterion = \"gini\")\n#forest_model.fit(x_train,y_train)\n#outputs for every model\nmodel3 = model_report(forest_model,X_SM,Y_SM,X_Test,Y_Test, \"Random Forest\")\nmodel3","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:44:17.167239Z","iopub.execute_input":"2023-04-07T06:44:17.168138Z","iopub.status.idle":"2023-04-07T06:49:08.526450Z","shell.execute_reply.started":"2023-04-07T06:44:17.168085Z","shell.execute_reply":"2023-04-07T06:49:08.525080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n### KNN\n\nfrom sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n           weights='uniform')\nmodel6= model_report(knn,X_SM,Y_SM,X_Test,Y_Test,\n                      \"KNN Classifier\")\nmodel6\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:49:08.532233Z","iopub.execute_input":"2023-04-07T06:49:08.533012Z","iopub.status.idle":"2023-04-07T06:50:22.724304Z","shell.execute_reply.started":"2023-04-07T06:49:08.532966Z","shell.execute_reply":"2023-04-07T06:50:22.723332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n### GaussianNB\n\nfrom sklearn.naive_bayes import GaussianNB\ngnb = GaussianNB(priors=None)\nmodel7 = model_report(gnb,X_SM,Y_SM,X_Test,Y_Test,\n                      \"Naive Bayes\")\nmodel7\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:50:22.725907Z","iopub.execute_input":"2023-04-07T06:50:22.726546Z","iopub.status.idle":"2023-04-07T06:50:22.895210Z","shell.execute_reply.started":"2023-04-07T06:50:22.726506Z","shell.execute_reply":"2023-04-07T06:50:22.894165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n### LightGBM\n\nlearning_rate = 0.1\nnum_leaves = 15\nmin_data_in_leaf = 50\nnum_boost_round = 30\n\n\nfrom lightgbm import LGBMClassifier\n\nlgbm_c = LGBMClassifier()\nmodel10 = model_report(lgbm_c,X_SM,Y_SM,X_Test,Y_Test,\n                      \"LGBM Classifier\")\n\nmodel10","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:50:22.897196Z","iopub.execute_input":"2023-04-07T06:50:22.897901Z","iopub.status.idle":"2023-04-07T06:50:25.322759Z","shell.execute_reply.started":"2023-04-07T06:50:22.897859Z","shell.execute_reply":"2023-04-07T06:50:25.321813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### XGBOOST\n\nfrom xgboost import XGBClassifier\nxgb_model = XGBClassifier( booster='gbtree', \n                 gamma=0, learning_rate=0.1,\n                    max_depth = 7, min_child_weight=1,  \n                    n_jobs=1,  objective='binary:logistic', \n                    reg_alpha=0, reg_lambda=1, scale_pos_weight=1,n_estimators=30)\n\n\n# xgb_model.fit(x_train,y_train)\n#outputs for every model\nmodel5 = model_report(xgb_model,X_SM,Y_SM,X_Test,Y_Test, \"XGBOOST\")\nmodel5","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:50:25.324334Z","iopub.execute_input":"2023-04-07T06:50:25.324977Z","iopub.status.idle":"2023-04-07T06:50:42.211273Z","shell.execute_reply.started":"2023-04-07T06:50:25.324936Z","shell.execute_reply":"2023-04-07T06:50:42.210272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# pred_xgb=xgb_model.predict(x_cv)\n# score_xgb = accuracy_score(pred_xgb,y_cv)*100\n# score_xgb\n\n# param_test = {\n#  'max_depth':[4,5,6],\n#  'min_child_weight':[4,5,6],\n#  'n_estimators':[100,150]\n# }\n# gsearch2 = GridSearchCV(estimator = XGBClassifier( learning_rate=0.1, \n#  min_child_weight=2, gamma=0, subsample=0.8, colsample_bytree=0.8,\n#  objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n#  param_grid = param_test, scoring='roc_auc',n_jobs=4, cv=5)\n# gsearch2.fit(x_train,y_train)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:50:42.212715Z","iopub.execute_input":"2023-04-07T06:50:42.214033Z","iopub.status.idle":"2023-04-07T06:50:42.220133Z","shell.execute_reply.started":"2023-04-07T06:50:42.213992Z","shell.execute_reply":"2023-04-07T06:50:42.218567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## Summary After Applying SMOTE\n","metadata":{}},{"cell_type":"code","source":"\n#concat all models\nmodel_performances = pd.concat([model1,model2,model3,\n                                model5,model6,\n                                model7,\n                                model10],axis = 0).reset_index()\n\nmodel_performances = model_performances.drop(columns = \"index\",axis =1)\n\nmodel_performances","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:58:18.866015Z","iopub.execute_input":"2023-04-07T06:58:18.866450Z","iopub.status.idle":"2023-04-07T06:58:18.896716Z","shell.execute_reply.started":"2023-04-07T06:58:18.866413Z","shell.execute_reply":"2023-04-07T06:58:18.895112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SMOTE OMEK","metadata":{}},{"cell_type":"code","source":"\n\nfrom imblearn.combine import SMOTETomek\nsm = SMOTETomek(random_state=1)\nX_SM, Y_SM = sm.fit_resample(X, y)\n\ny.value_counts().plot(kind='bar', title='Count of Target Classes in Dataset Before Sampling')\n\nY_SM.value_counts().plot(kind='bar', title='Count of Target Classes in Dataset After Sampling')\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:50:42.221935Z","iopub.execute_input":"2023-04-07T06:50:42.222450Z","iopub.status.idle":"2023-04-07T06:58:18.861477Z","shell.execute_reply.started":"2023-04-07T06:50:42.222414Z","shell.execute_reply":"2023-04-07T06:58:18.860076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###  Logistic regression \n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score\n\nlogistic_model = LogisticRegression(random_state=1)\n#outputs for every model\nmodel1 = model_report(logistic_model,X_SM,Y_SM,X_Test,Y_Test, \"Logistic Regression\")\n\nmodel1\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:58:18.898394Z","iopub.execute_input":"2023-04-07T06:58:18.898906Z","iopub.status.idle":"2023-04-07T06:58:20.040172Z","shell.execute_reply.started":"2023-04-07T06:58:18.898866Z","shell.execute_reply":"2023-04-07T06:58:20.038434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n### Decision Tree\n\nfrom sklearn.tree import DecisionTreeClassifier\ntree_model = DecisionTreeClassifier(max_depth = 9,\n                                       random_state = 123,\n                                       splitter  = \"best\",\n                                       criterion = \"gini\",\n                                      )\n#outputs for every model\nmodel2 = model_report(tree_model,X_SM,Y_SM,X_Test,Y_Test, \" Decision Tree\")\nmodel2\n\n### Random Forest\n\nfrom sklearn.ensemble import RandomForestClassifier\nforest_model = RandomForestClassifier(n_estimators = 1000,\n                             max_depth = 9,\n                             criterion = \"gini\")\n#forest_model.fit(x_train,y_train)\n#outputs for every model\nmodel3 = model_report(forest_model,X_SM,Y_SM,X_Test,Y_Test, \"Random Forest\")\nmodel3\n\n# pred_cv_forest=forest_model.predict(x_cv)\n# score_forest = accuracy_score(pred_cv_forest,y_cv)*100\n# score_forest\n\n# importances = pd.Series(forest_model.feature_importances_,index=X.columns)\n# importances.plot(kind='barh', figsize=(12,8))\n\n### KNN\n\nfrom sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n           weights='uniform')\nmodel6= model_report(knn,X_SM,Y_SM,X_Test,Y_Test,\n                      \"KNN Classifier\")\nmodel6\n\n\n\n### GaussianNB\n\nfrom sklearn.naive_bayes import GaussianNB\ngnb = GaussianNB(priors=None)\nmodel7 = model_report(gnb,X_SM,Y_SM,X_Test,Y_Test,\n                      \"Naive Bayes\")\nmodel7\n\n### LightGBM\n\nlearning_rate = 0.1\nnum_leaves = 15\nmin_data_in_leaf = 2000\nfeature_fraction = 0.6\nnum_boost_round = 10000\nparams = {\"objective\": \"binary\",\n          \"boosting_type\": \"gbdt\",\n          \"learning_rate\": learning_rate,\n          \"num_leaves\": num_leaves,\n           \"max_bin\": 256,\n          \"feature_fraction\": feature_fraction,\n          \"verbosity\": 0,\n          \"drop_rate\": 0.1,\n          \"is_unbalance\": False,\n          \"max_drop\": 50,\n          \"min_child_samples\": 10,\n          \"min_child_weight\": 150,\n          \"min_split_gain\": 0,\n          \"subsample\": 0.9\n          }\n\nfrom lightgbm import LGBMClassifier\n\nlgbm_c = LGBMClassifier(**params)\nmodel10 = model_report(lgbm_c,X_SM,Y_SM,X_Test,Y_Test,\n                      \"LGBM Classifier\")\n\nmodel10\n\nmodel10\n\n### XGBOOST\n\nfrom xgboost import XGBClassifier\nxgb_model = XGBClassifier( booster='gbtree', \n                 gamma=0, learning_rate=0.1,\n                    max_depth = 7, min_child_weight=1,  \n                    n_jobs=1,  objective='binary:logistic', \n                    reg_alpha=0, reg_lambda=1, scale_pos_weight=1,n_estimators=50)\n\n\n# xgb_model.fit(x_train,y_train)\n#outputs for every model\nmodel5 = model_report(xgb_model,X_SM,Y_SM,X_Test,Y_Test, \"XGBOOST\")\nmodel5\n\n# pred_xgb=xgb_model.predict(x_cv)\n# score_xgb = accuracy_score(pred_xgb,y_cv)*100\n# score_xgb\n\n# param_test = {\n#  'max_depth':[4,5,6],\n#  'min_child_weight':[4,5,6],\n#  'n_estimators':[100,150]\n# }\n# gsearch2 = GridSearchCV(estimator = XGBClassifier( learning_rate=0.1, \n#  min_child_weight=2, gamma=0, subsample=0.8, colsample_bytree=0.8,\n#  objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n#  param_grid = param_test, scoring='roc_auc',n_jobs=4, cv=5)\n# gsearch2.fit(x_train,y_train)\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:58:20.046448Z","iopub.execute_input":"2023-04-07T06:58:20.049212Z","iopub.status.idle":"2023-04-07T07:04:29.572139Z","shell.execute_reply.started":"2023-04-07T06:58:20.049144Z","shell.execute_reply":"2023-04-07T07:04:29.570383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary After Applying SMOTE OMEK","metadata":{}},{"cell_type":"code","source":"\n\n#concat all models\nmodel_performances = pd.concat([model1,model2,model3,\n                                model5,model6,\n                                model7,\n                                model10],axis = 0).reset_index()\n\nmodel_performances = model_performances.drop(columns = \"index\",axis =1)\n\nmodel_performances\n","metadata":{"execution":{"iopub.status.busy":"2023-04-07T07:04:29.574258Z","iopub.execute_input":"2023-04-07T07:04:29.574842Z","iopub.status.idle":"2023-04-07T07:04:29.619514Z","shell.execute_reply.started":"2023-04-07T07:04:29.574769Z","shell.execute_reply":"2023-04-07T07:04:29.617744Z"},"trusted":true},"execution_count":null,"outputs":[]}]}