{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \ntest = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\ntrain = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\n\n# Random permutation is needed for CatBoostEncoder to reduce leakage\ndef random_permutation(x):\n    perm = np.random.permutation(len(x)) \n    x = x.iloc[perm].reset_index(drop=True) \n    return x\n\ntrain = random_permutation(train)\ntest = random_permutation(test)\n\ntrain_ids = train.id\ntest_ids = test.id\n\ntrain.drop('id', 1, inplace=True)\ntest.drop('id', 1, inplace=True)\n\ntrain_targets = train.target\ntrain.drop('target', 1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"Preprocessing strategy:\n\n* For high-cardinality features (`nom_5` to `nom_9` and `ord_5`) use `CatBoostEncoder` target encoding (it performs a type of leave-one-out encoding)\n* For other features use `OneHotEncoder`"},{"metadata":{"trusted":true},"cell_type":"code","source":"from category_encoders.cat_boost import CatBoostEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.compose import ColumnTransformer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Missing values\n\nConvert columns to strings and replace NAs with a non-NaN value to ensure that the one-hot encoder will treat missing values as their own class."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfor col in train.columns:\n    train[col] = train[col].astype(str)\n    train[col].fillna('NA', inplace=True)\n    test[col] = test[col].astype(str)\n    test[col].fillna('NA', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Encoders"},{"metadata":{},"cell_type":"markdown","source":"### CatBoostEncoder for high-cardinality features"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# noms 5-9\nfor i in [5,6,7,8,9]:\n    cbe = CatBoostEncoder()\n    train[f'nom_{i}'] = cbe.fit_transform(train[f'nom_{i}'], train_targets)\n    test[f'nom_{i}'] = cbe.transform(test[f'nom_{i}'])\n\n# ord 5\ncbe = CatBoostEncoder()\ntrain['ord_5'] = cbe.fit_transform(train['ord_5'], train_targets)\ntest['ord_5'] = cbe.transform(test['ord_5'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### OneHotEncoder for other features"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nohe_cols = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4',\n            'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4',\n            'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4',\n            'day', 'month']\n\n# ColumnTransformer enables applying OneHotEncoder to the entire dataframe\ntransformer = ColumnTransformer(\n    [\n        (\"ohe\",\n         OneHotEncoder(sparse=True, drop='first'),\n         ohe_cols\n         )\n    ], remainder='passthrough'\n)\ntrain = transformer.fit_transform(train)\ntest = transformer.fit_transform(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Logistic Regression\n\nI'll use LogisticRegressionCV to do automatic parameter selection, and give me a sense of cross-validated model performance."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfrom sklearn.linear_model import LogisticRegressionCV\nclf = LogisticRegressionCV(cv=5, \n                           scoring='roc_auc', \n                           random_state=42, \n                           verbose=True, \n                           n_jobs=-1,\n                           max_iter = 1000)\nclf.fit(train, train_targets)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model score averaged over folds"},{"metadata":{"trusted":true},"cell_type":"code","source":"np.mean(clf.scores_[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = clf.predict_proba(test)[:, 1]\npreds = pd.DataFrame(list(zip(test_ids, preds)), columns = ['id', 'target'])\npreds.sort_values(by=['id'], inplace = True)\n\npreds.to_csv(\"./my_submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}