{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \ntest = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\ntrain = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\n\n# Random permutation is needed for CatBoostEncoder to reduce leakage\ndef random_permutation(x):\n    perm = np.random.permutation(len(x)) \n    x = x.iloc[perm].reset_index(drop=True) \n    return x\n\ntrain = random_permutation(train)\ntest = random_permutation(test)\n\ntrain_ids = train.id\ntest_ids = test.id\n\ntrain.drop('id', 1, inplace=True)\ntest.drop('id', 1, inplace=True)\n\ntrain_targets = train.target\ntrain.drop('target', 1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"## Mapping values\n\nFor the binary and ordinal variables I will simply use a value mapping approach."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# bin variables\nbin_recode = {0: 0, 1: 1, 'F':0, 'T':1, 'N':0, 'Y':1}\nfor i in range(5):\n    train[f'bin_{i}'] = train[f'bin_{i}'].map(bin_recode)\n    test[f'bin_{i}'] = test[f'bin_{i}'].map(bin_recode)\n\n# ord_1\nlevels = { 'Novice':0, 'Contributor':1, \n          'Expert':2, 'Master':3, 'Grandmaster':4 }\ntrain['ord_1'] = train['ord_1'].map(levels)\ntest['ord_1'] = test['ord_1'].map(levels)\n\n# ord_2\ntemps = { 'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, \n         'Boiling Hot':4, 'Lava Hot':5 }\ntrain['ord_2'] = train['ord_2'].map(temps)\ntest['ord_2'] = test['ord_2'].map(temps)\n\n# ord_3\nlowercase_letters = {'a':1,'b':2,'c':3,'d':4,'e':5,'f':6,\n                     'g':7,'h':8,'i':9,'j':10,'k':11,\n                     'l':12,'m':13,'n':14,'o':15,'p':16,\n                     'q':17,'r':18,'s':19,'t':20,'u':21,\n                     'v':22,'w':23,'x':24,'y':25,'z':26}\ntrain['ord_3'] = train['ord_3'].map(lowercase_letters)\ntest['ord_3'] = test['ord_3'].map(lowercase_letters)\n\n# ord_4\nuppercase_letters = {'A':1,'B':2,'C':3,'D':4,'E':5,'F':6,\n                     'G':7,'H':8,'I':9,'J':10,'K':11,\n                     'L':12,'M':13,'N':14,'O':15,'P':16,\n                     'Q':17,'R':18,'S':19,'T':20,'U':21,\n                     'V':22,'W':23,'X':24,'Y':25,'Z':26}\ntrain['ord_4'] = train['ord_4'].map(uppercase_letters)\ntest['ord_4'] = test['ord_4'].map(uppercase_letters)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dummy coding\n\nFor `nom_0` to `nom_4` I will use dummy coding."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nnoms_0_4 = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\ntrain = pd.get_dummies(train, \n                       columns = noms_0_4, \n                       prefix = noms_0_4,\n                       drop_first=True,\n                       sparse=True, \n                       dtype=np.int8)\n\ntest = pd.get_dummies(test, \n                      columns = noms_0_4, \n                      prefix = noms_0_4,\n                      drop_first=True,\n                      sparse=True, \n                      dtype=np.int8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Target encoding\n\n`nom_5` to `nom_9` and `ord_5` are high-cardinality features. One way to handle these features is to use target encoding. However, there are different ways of doing target encoding and some are better at avoiding leakage / overfitting. Here I use the CatBoostEncoder which implements a leave-one-out strategy of target encoding."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfrom category_encoders.cat_boost import CatBoostEncoder\n\n# noms 5-9\nfor i in [5,6,7,8,9]:\n    cbe = CatBoostEncoder()\n    train[f'nom_{i}'] = cbe.fit_transform(train[f'nom_{i}'], train_targets)\n    test[f'nom_{i}'] = cbe.transform(test[f'nom_{i}'])\n\n# ord 5\ncbe = CatBoostEncoder()\ntrain['ord_5'] = cbe.fit_transform(train['ord_5'], train_targets)\ntest['ord_5'] = cbe.transform(test['ord_5'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CatBoost Classifier"},{"metadata":{},"cell_type":"markdown","source":"I used `grid_search()` to identify the best parameters for the `CatBoostClassifier`."},{"metadata":{"trusted":true},"cell_type":"code","source":"#grid = {'learning_rate': [0.1, 0.15, 0.2, 0.25, 0.3],\n#        'depth': [3, 4, 5, 6],\n#        'l2_leaf_reg': [1, 2, 3, 4, 5, 6, 7]}\n\n#grid_search_result = cb.grid_search(grid, \n#                                    X=train_cbe, \n#                                    y=train_targets, \n#                                    plot=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The model below uses the best parameters that I found."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfrom catboost import CatBoostClassifier\ncb = CatBoostClassifier(eval_metric='AUC',\n                        learning_rate=0.1,\n                        depth=3,\n                        l2_leaf_reg=5)\n\ncb.fit(train, train_targets, verbose=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = cb.predict_proba(test)[:, 1]\npreds_df = pd.DataFrame(list(zip(test_ids, preds)), \n                        columns = ['id', 'target'])\npreds_df.sort_values(by=['id'], inplace = True)\n\npreds_df.to_csv(\"./submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}