{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import the models and necessary libraries\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\n\n# use for pipeline and encode features\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.impute import SimpleImputer\n\n# models\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.cluster import KMeans\nimport sklearn.metrics.cluster as smc\n\n# validation of the models\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# import the datasets\ntrain = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ntest = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# can uncomment the following to look at the info (type) of the features, shape of the datasets\n#print(train.info())\n#print(test.info())\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Change boolean value to int so as to encode\ntrain['bin_3'] = train['bin_3'].apply(lambda x: 1 if x=='T' else 0)\ntrain['bin_4'] = train['bin_4'].apply(lambda x:1 if x =='Y' else 0)\ntest['bin_3'] = test['bin_3'].apply(lambda x:1 if x=='T' else 0)\ntest['bin_4'] = test['bin_4'].apply(lambda x:1 if x == 'Y' else 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# input the test_labels to validate the test sets later on\n# drop the target column in train sets to separate the features and values we need to predict\ntest_labels = train['target']\ntrain = train.drop(['target'],axis=1)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# uncomment this if you want to test a new model using smaller datasets\n# notice these are not randomly chosen, so be careful of overfitting\n# X_train_part = X_train[:4200]\n# y_train_part = y_train[:4200]\n# X_test_part = X_test[:1800]\n# y_test_part = y_test[:1800]\n#train_part = train[:6000]\n#test_labels2 = test_labels[:6000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pipelining the categorical features\n# I chose to do one hot encoder on some of the categorical features since they are type-A/type-B features\n# dropping some of the features out because they have high cardinalities, which would make the datasets have too many columns\n\n%%time\nfrom category_encoders.m_estimate import MEstimateEncoder\nimputer1 = SimpleImputer(strategy=\"median\")\nimputer = SimpleImputer(strategy='most-frequent')\ntrain_1=train\ndef Preparation(train,test_set=False):\n    \n    train_cat = train.drop([\"id\",\"nom_5\",\"nom_6\",\"nom_9\"],axis=1)\n    cat_pipeline = Pipeline([\n                ('imputer2',SimpleImputer(strategy='most_frequent')),\n                ('cat',OneHotEncoder(categories='auto')),\n                #('cat',MEstimateEncoder(verbose=0, cols=None, drop_invariant=False, return_df=True, handle_unknown='value', handle_missing='value', random_state=None, randomized=False, sigma=0.05, m=1.0)),\n    ])\n    train_cat_tr = cat_pipeline.fit_transform(train_cat)\n    categorical_features = list(train_cat)\n    \n    full_pipeline = ColumnTransformer([\n            #(\"num\", num_pipeline, numerical_features),\n            (\"cat\", cat_pipeline, categorical_features),\n        ])\n\n    train_prepared = full_pipeline.fit_transform(train)\n    print(train_prepared.shape)\n    return train_prepared\ntrain_1 = Preparation(train_1) #train_1\n#print(train_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# separate the datasets into 80% train sets and 20% test sets\n# can also do K-fold validation\n\nX_train,X_test,y_train,y_test = train_test_split(train_1,test_labels,random_state=42,test_size=0.2) #train_1,test_labels\n#print(help(train_test_split))\nprint(X_train.shape,X_test.shape,y_train.shape,y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Input the catboost models\n# can use loops or grid-searchCV to tune parameters\n\n%%time\nfrom catboost import CatBoostClassifier\nparams = { #30,2000，0.15,5\n    'bagging_temperature': 0.8,'l2_leaf_reg': 30,'iterations': 998,'learning_rate': 0.15,'depth': 5,\n    'random_strength': 0.8,'loss_function': 'Logloss','eval_metric': 'AUC','verbose': False\n}\ncatb = CatBoostClassifier(**params, nan_mode='Min').fit(X_train, y_train,verbose_eval=100, early_stopping_rounds=50,eval_set=(X_test, y_test),\n                                                        use_best_model=False,\n                                                        plot=True)\npreds2 = catb.predict_proba(X_test)[:,1]\n\nprint(\"ROC AUC score is %.4f\" %(roc_auc_score(y_test,preds2)))\n\nprint(\"Catboost Model Performance Results:\\n\")\nplot_roc_curve(catb,X_test,y_test)\nplt.title('ROC Curve')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission\ntest_id = test.index\ntest_sub = Preparation(test)\ntest_pred = catb.predict_proba(test_sub)[:,1]\nsubmission = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/sample_submission.csv\")\nsubmission.target = test_pred\nsubmission.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}