{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n

Classification coding challenge 🚩

\n
","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\nimport numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:58:05.170054Z","iopub.execute_input":"2021-11-25T04:58:05.170931Z","iopub.status.idle":"2021-11-25T04:58:06.170387Z","shell.execute_reply.started":"2021-11-25T04:58:05.170806Z","shell.execute_reply":"2021-11-25T04:58:06.169109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
1. Simple view of data
\n
","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:58:06.172486Z","iopub.execute_input":"2021-11-25T04:58:06.172763Z","iopub.status.idle":"2021-11-25T04:58:09.951386Z","shell.execute_reply.started":"2021-11-25T04:58:06.172729Z","shell.execute_reply":"2021-11-25T04:58:09.949921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:58:09.952756Z","iopub.execute_input":"2021-11-25T04:58:09.953051Z","iopub.status.idle":"2021-11-25T04:58:12.329367Z","shell.execute_reply.started":"2021-11-25T04:58:09.952991Z","shell.execute_reply":"2021-11-25T04:58:12.328292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **bin----Duality**\n- **nom----Nominal characteristics**\n- **ord----Sequence feature**\n- **day or month ----Date characteristics**","metadata":{}},{"cell_type":"code","source":"df_train.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:58:12.330522Z","iopub.execute_input":"2021-11-25T04:58:12.330747Z","iopub.status.idle":"2021-11-25T04:58:14.027491Z","shell.execute_reply.started":"2021-11-25T04:58:12.330720Z","shell.execute_reply":"2021-11-25T04:58:14.026252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"round(df_train.isnull().sum()/df_train.shape[0],2)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:58:14.029686Z","iopub.execute_input":"2021-11-25T04:58:14.029914Z","iopub.status.idle":"2021-11-25T04:58:15.157308Z","shell.execute_reply.started":"2021-11-25T04:58:14.029887Z","shell.execute_reply":"2021-11-25T04:58:15.156087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **null_num percent--3%**","metadata":{}},{"cell_type":"markdown","source":"
\n
2. Dealing with missing values
\n
\n \n
\n

💥My Choice：

\n \n
\n","metadata":{}},{"cell_type":"code","source":"binc = [\"bin_0\",\"bin_1\",\"bin_2\",\"bin_3\",\"bin_4\"]\nnomc = [\"nom_0\",\"nom_1\",\"nom_2\",\"nom_3\",\"nom_4\",\"nom_5\",\"nom_6\",\"nom_7\",\"nom_8\",\"nom_9\"]\nordc = [\"ord_0\",\"ord_1\",\"ord_2\",\"ord_3\",\"ord_4\",\"ord_5\"]\ndatec = [\"day\",\"month\"]\nmisscol = binc + ordc\n\ndef fill_na(data,colist):\n \n if len(colist)==10:# fill nom_\n for col in colist:\n data[col] = data[col].fillna(data[col].value_counts(normalize=True).idxmin())\n elif len(colist)==11: #fill_binc+ordc\n for col in colist:\n data[col] = data[col].fillna(\"Missing\")\n else:\n for col in colist:\n data[col] = data[col].fillna(data[col].median())\n return data","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:58:15.159080Z","iopub.execute_input":"2021-11-25T04:58:15.159396Z","iopub.status.idle":"2021-11-25T04:58:15.181263Z","shell.execute_reply.started":"2021-11-25T04:58:15.159364Z","shell.execute_reply":"2021-11-25T04:58:15.180480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = fill_na(df_train,misscol)\ndf_train = fill_na(df_train,nomc)\ndf_train = fill_na(df_train,datec)\n#---------------------------------\ndf_test = fill_na(df_test,misscol)\ndf_test = fill_na(df_test,nomc)\ndf_test = fill_na(df_test,datec)\nprint('df_train null_num: ',df_train.isnull().all().sum())\nprint('df_test null num: ',df_test.isnull().all().sum())\nprint(df_train.head())","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:58:15.182593Z","iopub.execute_input":"2021-11-25T04:58:15.182929Z","iopub.status.idle":"2021-11-25T04:58:21.455696Z","shell.execute_reply.started":"2021-11-25T04:58:15.182895Z","shell.execute_reply":"2021-11-25T04:58:21.454879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
3. Encoding process
\n
\n

💦Due to the use of algorithms such as XGboost, one-hot encoding is avoided as much as possible, and features are transformed according to label encoding.\n At the same time, for features containing letters and values, the corresponding letters and numbers will be extracted as new column identifiers.\n \n

\n
\n
","metadata":{}},{"cell_type":"code","source":"for col in binc:\n print(\"This is {} unqiue values:\".format(col))\n print(df_train[col].unique())\n print(\"-\"*25)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:58:21.456985Z","iopub.execute_input":"2021-11-25T04:58:21.457255Z","iopub.status.idle":"2021-11-25T04:58:21.810762Z","shell.execute_reply.started":"2021-11-25T04:58:21.457225Z","shell.execute_reply":"2021-11-25T04:58:21.809475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- dealing wirh bin_columns","metadata":{}},{"cell_type":"code","source":"bin_num = {\"Missing\":0.0,0.0:1.0,1.0:2.0}\nbin_3 = {\"Missing\":0.0,\"F\":1.0,\"T\":2.0}\nbin_4 = {\"Missing\":0.0,\"N\":1.0,\"Y\":2.0}\ndf_train[\"bin_0\"] = df_train[\"bin_0\"].map(bin_num)\ndf_train[\"bin_1\"] = df_train[\"bin_1\"].map(bin_num)\ndf_train[\"bin_2\"] = df_train[\"bin_2\"].map(bin_num)\ndf_train[\"bin_3\"] = df_train[\"bin_3\"].map(bin_3)\ndf_train[\"bin_4\"] = df_train[\"bin_4\"].map(bin_4)\n#-------------------------------------------------\ndf_test[\"bin_0\"] = df_test[\"bin_0\"].map(bin_num)\ndf_test[\"bin_1\"] = df_test[\"bin_1\"].map(bin_num)\ndf_test[\"bin_2\"] = df_test[\"bin_2\"].map(bin_num)\ndf_test[\"bin_3\"] = df_test[\"bin_3\"].map(bin_3)\ndf_test[\"bin_4\"] = df_test[\"bin_4\"].map(bin_4)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-25T04:58:21.812371Z","iopub.execute_input":"2021-11-25T04:58:21.812610Z","iopub.status.idle":"2021-11-25T04:58:22.893470Z","shell.execute_reply.started":"2021-11-25T04:58:21.812582Z","shell.execute_reply":"2021-11-25T04:58:22.892269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ordc:\n print(\"This is {} unqiue values:\".format(col))\n print(df_train[col].unique())\n print(\"-\"*60)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-25T04:58:22.895076Z","iopub.execute_input":"2021-11-25T04:58:22.895401Z","iopub.status.idle":"2021-11-25T04:58:23.251440Z","shell.execute_reply.started":"2021-11-25T04:58:22.895367Z","shell.execute_reply":"2021-11-25T04:58:23.250303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nord_0 = {\"Missing\":0.0}\nord_1 = {\"Missing\":0.0,\"Novice\":1.0,\"Contributor\":2.0,\"Expert\":3.0,\"Master\":4.0,\"Grandmaster\":5.0}\nord_2 = {\"Missing\":0.0,\"Freezing\":1.0,\"Cold\":2.0,\"Warm\":3.0,\"Hot\":4.0,\"Boiling Hot\":5.0,\"Lava Hot\":6.0}\nord_3 = {\"Missing\":0.0,\"a\":1.0,\"b\":2.0,\"c\":3.0,\"d\":4.0,\"e\":5.0,\"f\":6.0,\"g\":7.0,\"h\":8.0,\"i\":9.0,\"j\":10.0,\"k\":11.0,\"l\":12.0,\"m\":13.0,\"n\":14.0,\"o\":15.0}\nord_4 = {\"Missing\":0.0,\"A\":1.0,\"B\":2.0,\"C\":3.0,\"D\":4.0,\"E\":5.0,\"F\":6.0,\"G\":7.0,\"H\":8.0,\"I\":9.0,\"J\":10.0,\"K\":11.0,\"L\":12.0,\n \"M\":13.0,\"N\":14.0,\"O\":15.0,\"P\":16.0,\"Q\":17.0,\"R\":18.0,\"S\":19.0,\"T\":20.0,\"U\":21.0,\"V\":22.0,\"W\":23.0,\"X\":24.0,\"Y\":25.0,\"Z\":26.0}\n\nlab = LabelEncoder()\ndf_train[\"ord_0\"] = df_train[\"ord_0\"].replace(ord_0)\ndf_train[\"ord_1\"] = df_train[\"ord_1\"].map(ord_1)\ndf_train[\"ord_2\"] = df_train[\"ord_2\"].map(ord_2)\ndf_train[\"ord_3\"] = df_train[\"ord_3\"].map(ord_3)\ndf_train[\"ord_4\"] = df_train[\"ord_4\"].map(ord_4)\nlab.fit(df_train[\"ord_5\"].values)\ndf_train['ord_5'] = lab.transform(df_train[\"ord_5\"])\n\n\ndf_test[\"ord_0\"] = df_test[\"ord_0\"].replace(ord_0)\ndf_test[\"ord_1\"] = df_test[\"ord_1\"].map(ord_1)\ndf_test[\"ord_2\"] = df_test[\"ord_2\"].map(ord_2)\ndf_test[\"ord_3\"] = df_test[\"ord_3\"].map(ord_3)\ndf_test[\"ord_4\"] = df_test[\"ord_4\"].map(ord_4)\ndf_test['ord_5'] = lab.transform(df_test[\"ord_5\"])\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-25T04:58:23.252901Z","iopub.execute_input":"2021-11-25T04:58:23.253148Z","iopub.status.idle":"2021-11-25T04:58:25.332567Z","shell.execute_reply.started":"2021-11-25T04:58:23.253121Z","shell.execute_reply":"2021-11-25T04:58:25.331724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in nomc:\n print(\"This is {} unqiue values:\".format(col))\n print(df_train[col].unique())\n print(\"-\"*60)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-25T04:58:25.333753Z","iopub.execute_input":"2021-11-25T04:58:25.334047Z","iopub.status.idle":"2021-11-25T04:58:25.920449Z","shell.execute_reply.started":"2021-11-25T04:58:25.334011Z","shell.execute_reply":"2021-11-25T04:58:25.919503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lab1 = LabelEncoder()\nlab1.fit(df_train[\"nom_0\"])\ndf_train[\"nom_0\"] = lab1.transform(df_train[\"nom_0\"])\ndf_test[\"nom_0\"] = lab1.transform(df_test[\"nom_0\"])\n\nlab2 = LabelEncoder()\nlab2.fit(df_train[\"nom_1\"])\ndf_train[\"nom_1\"] = lab2.transform(df_train[\"nom_1\"])\ndf_test[\"nom_1\"] = lab2.transform(df_test[\"nom_1\"])\n\nlab3 = LabelEncoder()\nlab3.fit(df_train[\"nom_2\"])\ndf_train[\"nom_2\"] = lab3.transform(df_train[\"nom_2\"])\ndf_test[\"nom_2\"] = lab3.transform(df_test[\"nom_2\"])\n\nlab4 = LabelEncoder()\nlab4.fit(df_train[\"nom_3\"])\ndf_train[\"nom_3\"] = lab4.transform(df_train[\"nom_3\"])\ndf_test[\"nom_3\"] = lab4.transform(df_test[\"nom_3\"])\n\nlab5 = LabelEncoder()\nlab5.fit(df_train[\"nom_4\"])\ndf_train[\"nom_4\"] = lab5.transform(df_train[\"nom_4\"])\ndf_test[\"nom_4\"] = lab5.transform(df_test[\"nom_4\"])\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-25T04:58:25.922090Z","iopub.execute_input":"2021-11-25T04:58:25.922437Z","iopub.status.idle":"2021-11-25T04:58:28.199036Z","shell.execute_reply.started":"2021-11-25T04:58:25.922400Z","shell.execute_reply":"2021-11-25T04:58:28.197819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Regarding nom_5 to nom_9 as a password, calculate the length of distinct values.","metadata":{}},{"cell_type":"code","source":"import re\ndf_train[\"nom_5\"] = df_train[\"nom_5\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_train[\"nom_6\"] = df_train[\"nom_6\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_train[\"nom_7\"] = df_train[\"nom_7\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_train[\"nom_8\"] = df_train[\"nom_8\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_train[\"nom_9\"] = df_train[\"nom_9\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\n#---------------------------------------------------------------------------------------------------------------\ndf_test[\"nom_5\"] = df_test[\"nom_5\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_test[\"nom_6\"] = df_test[\"nom_6\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_test[\"nom_7\"] = df_test[\"nom_7\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_test[\"nom_8\"] = df_test[\"nom_8\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_test[\"nom_9\"] = df_test[\"nom_9\"].apply(lambda x:len(set(re.sub(r'\\D',\"\",x))) + len(set(re.sub(r'\\d',\"\",x))))\ndf_train.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-25T04:58:28.202141Z","iopub.execute_input":"2021-11-25T04:58:28.202573Z","iopub.status.idle":"2021-11-25T04:59:03.972898Z","shell.execute_reply.started":"2021-11-25T04:58:28.202536Z","shell.execute_reply":"2021-11-25T04:59:03.972053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
4. Feature Screening
\n
","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\ncorr = df_train.corr()\nsns.heatmap(corr,annot=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:59:03.974041Z","iopub.execute_input":"2021-11-25T04:59:03.974283Z","iopub.status.idle":"2021-11-25T04:59:09.291467Z","shell.execute_reply.started":"2021-11-25T04:59:03.974253Z","shell.execute_reply":"2021-11-25T04:59:09.290455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\ntarget = df_train[[\"target\"]]\nfeature = df_train.drop(columns=[\"target\",\"id\"])\nrf = RandomForestClassifier()\nrf.fit(feature,target.values.reshape(-1,1))\nim = pd.DataFrame()\nplt.figure(figsize=(15,10))\ng = sns.barplot(y=rf.feature_importances_,x=feature.columns,edgecolor=\"black\",linewidth=3,color=\"#87ceed\")\ng.spines[['top', 'right','bottom']].set_visible(False)\ng.axhline(0.02,color='r',linewidth=1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T04:59:09.293551Z","iopub.execute_input":"2021-11-25T04:59:09.294392Z","iopub.status.idle":"2021-11-25T05:01:15.273227Z","shell.execute_reply.started":"2021-11-25T04:59:09.294328Z","shell.execute_reply":"2021-11-25T05:01:15.271760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **According to the importance features of random forest screening, we plan to eliminate the feature columns from bin_0 to bin_2.**","metadata":{}},{"cell_type":"code","source":"feature = feature.drop(columns=[\"bin_0\",\"bin_1\",\"bin_2\"])\ndf_test = df_test.drop(columns=[\"bin_0\",\"bin_1\",\"bin_2\"])","metadata":{"execution":{"iopub.status.busy":"2021-11-25T05:01:15.274763Z","iopub.execute_input":"2021-11-25T05:01:15.275026Z","iopub.status.idle":"2021-11-25T05:01:15.457605Z","shell.execute_reply.started":"2021-11-25T05:01:15.274977Z","shell.execute_reply":"2021-11-25T05:01:15.456066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T05:01:15.459594Z","iopub.execute_input":"2021-11-25T05:01:15.459990Z","iopub.status.idle":"2021-11-25T05:01:15.488401Z","shell.execute_reply.started":"2021-11-25T05:01:15.459944Z","shell.execute_reply":"2021-11-25T05:01:15.486954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **It can be seen that the data is seriously unbalanced, so when dividing the data and the training data, use hierarchical cross-validation.**","metadata":{}},{"cell_type":"markdown","source":"
\n
5 Model XGBoost and RandomForest
\n
","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold,cross_val_score\nfrom sklearn.metrics import classification_report\nimport xgboost as xgb \nstf = StratifiedKFold(random_state=123,shuffle=False)\nfor train_index,test_index in stf.split(feature,target):\n x_train,x_test = feature.iloc[train_index],feature.iloc[test_index]\n y_train,y_test = target.iloc[train_index],target.iloc[test_index]\nxgb = xgb.XGBClassifier(learning_rate=0.1, n_estimators=400, max_depth=5, min_child_weight=1, gamma=0,\n subsample=0.8, colsample_bytree=0.8, objective= 'binary:logistic', nthread=4, seed=27)\nxgb.fit(x_train,y_train)\npre_xgb = xgb.predict(x_test)\nxgb_prob = xgb.predict_proba(x_test)[:,1]","metadata":{"execution":{"iopub.status.busy":"2021-11-25T05:43:58.181897Z","iopub.execute_input":"2021-11-25T05:43:58.183139Z","iopub.status.idle":"2021-11-25T05:45:58.333757Z","shell.execute_reply.started":"2021-11-25T05:43:58.183091Z","shell.execute_reply":"2021-11-25T05:45:58.332913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test,pre_xgb))","metadata":{"execution":{"iopub.status.busy":"2021-11-25T05:45:58.507568Z","iopub.execute_input":"2021-11-25T05:45:58.507864Z","iopub.status.idle":"2021-11-25T05:45:58.681164Z","shell.execute_reply.started":"2021-11-25T05:45:58.507824Z","shell.execute_reply":"2021-11-25T05:45:58.679903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_1 = RandomForestClassifier()\nrf_1.fit(x_train,y_train)\nrf_1_pre = rf_1.predict(x_test)\nrf_prob = rf_1.predict_proba(x_test)[:,1]\nprint(classification_report(y_test,rf_1_pre))","metadata":{"execution":{"iopub.status.busy":"2021-11-25T05:49:02.468630Z","iopub.execute_input":"2021-11-25T05:49:02.469090Z","iopub.status.idle":"2021-11-25T05:50:56.547285Z","shell.execute_reply.started":"2021-11-25T05:49:02.469041Z","shell.execute_reply":"2021-11-25T05:50:56.546092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import ConfusionMatrixDisplay,confusion_matrix\nfig,ax = plt.subplots(1,2,figsize=(10,6))\ncmxg = confusion_matrix(y_test,pre_xgb) #xgboost\ndisp = ConfusionMatrixDisplay(cmxg)\ndisp.plot(ax=ax[0])\n\n\ncm = confusion_matrix(y_test,rf_1_pre) #RF\ndis = ConfusionMatrixDisplay(cm)\ndis.plot(ax=ax[1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T05:50:56.549737Z","iopub.execute_input":"2021-11-25T05:50:56.550013Z","iopub.status.idle":"2021-11-25T05:50:57.392240Z","shell.execute_reply.started":"2021-11-25T05:50:56.549970Z","shell.execute_reply":"2021-11-25T05:50:57.390885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Xgboost on the left, random forest on the right.**\n- **The actual number is 0, and the number of predictions is 0. Random forest is stronger than xgboost.**\n- **The actual number is 1, and the number of predictions is 1. Xgboost forest is stronger than Random forest.**\n\n","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nprint(\"xgboost roc auc score:\",roc_auc_score(y_test,xgb_prob))\nprint(\"RF roc auc score:\",roc_auc_score(y_test,rf_prob))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-25T05:51:55.867774Z","iopub.execute_input":"2021-11-25T05:51:55.868383Z","iopub.status.idle":"2021-11-25T05:51:55.972709Z","shell.execute_reply.started":"2021-11-25T05:51:55.868335Z","shell.execute_reply":"2021-11-25T05:51:55.971719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n
6 Stack
\n
","metadata":{}},{"cell_type":"code","source":"from mlxtend.classifier import StackingClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom mlxtend.plotting import plot_decision_regions\nimport matplotlib.gridspec as gridspec\n\nLR = LogisticRegression()\nstack = StackingClassifier(classifiers=[xgb,rf_1],meta_classifier=LR)\nstack.fit(x_train,y_train)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-25T06:00:02.482235Z","iopub.execute_input":"2021-11-25T06:00:02.482817Z","iopub.status.idle":"2021-11-25T06:04:13.110182Z","shell.execute_reply.started":"2021-11-25T06:00:02.482764Z","shell.execute_reply":"2021-11-25T06:04:13.109082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stack_pre = stack.predict(x_test)\nstack_prob = stack.predict_proba(x_test)[:,1]","metadata":{"execution":{"iopub.status.busy":"2021-11-25T06:24:51.717516Z","iopub.execute_input":"2021-11-25T06:24:51.717844Z","iopub.status.idle":"2021-11-25T06:25:05.043564Z","shell.execute_reply.started":"2021-11-25T06:24:51.717809Z","shell.execute_reply":"2021-11-25T06:25:05.042450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test,stack_pre))","metadata":{"execution":{"iopub.status.busy":"2021-11-25T06:26:19.300954Z","iopub.execute_input":"2021-11-25T06:26:19.301514Z","iopub.status.idle":"2021-11-25T06:26:19.472655Z","shell.execute_reply.started":"2021-11-25T06:26:19.301479Z","shell.execute_reply":"2021-11-25T06:26:19.471794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import ConfusionMatrixDisplay,confusion_matrix\nfig,ax = plt.subplots(1,2,figsize=(16,6))\n\n\ncmxg = confusion_matrix(y_test,pre_xgb) #xgboost\ndisp = ConfusionMatrixDisplay(cmxg)\ndisp.plot(ax=ax[0])\n\n\ncm = confusion_matrix(y_test,rf_1_pre) #RF\ndis = ConfusionMatrixDisplay(cm)\ndis.plot(ax=ax[1])\nplt.show()\n\n\nscm = confusion_matrix(y_test,stack_pre) #stack\ndistack = ConfusionMatrixDisplay(scm)\ndistack.plot()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-25T06:32:50.722185Z","iopub.execute_input":"2021-11-25T06:32:50.722521Z","iopub.status.idle":"2021-11-25T06:32:51.909462Z","shell.execute_reply.started":"2021-11-25T06:32:50.722490Z","shell.execute_reply":"2021-11-25T06:32:51.908313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"stack roc auc score:\",roc_auc_score(y_test,stack_prob))","metadata":{"execution":{"iopub.status.busy":"2021-11-25T06:33:45.987335Z","iopub.execute_input":"2021-11-25T06:33:45.987663Z","iopub.status.idle":"2021-11-25T06:33:46.018048Z","shell.execute_reply.started":"2021-11-25T06:33:45.987630Z","shell.execute_reply":"2021-11-25T06:33:46.017457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Score decreased**","metadata":{}},{"cell_type":"code","source":"ids = df_test[[\"id\"]]\ntest_data = df_test.drop(columns=[\"id\"])\nsubmisson_pre = stack.predict(test_data.values)\nsub = pd.DataFrame()\nsub[\"id\"] = ids\nsub[\"target\"] = submisson_pre\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-25T06:40:07.052540Z","iopub.execute_input":"2021-11-25T06:40:07.052969Z","iopub.status.idle":"2021-11-25T06:40:26.555431Z","shell.execute_reply.started":"2021-11-25T06:40:07.052937Z","shell.execute_reply":"2021-11-25T06:40:26.554441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\",index=None)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T06:40:53.873855Z","iopub.execute_input":"2021-11-25T06:40:53.874150Z","iopub.status.idle":"2021-11-25T06:40:54.622632Z","shell.execute_reply.started":"2021-11-25T06:40:53.874121Z","shell.execute_reply":"2021-11-25T06:40:54.621685Z"},"trusted":true},"execution_count":null,"outputs":[]}]}