{"cells":[{"metadata":{"_uuid":"5b2a7a0e174c4078e55c5e1a85d7311dd755460f"},"cell_type":"markdown","source":"# Notes\n\n* Getting started with CatBoost as a single model for training"},{"metadata":{"_uuid":"fe07fa0a56f2df7a043760fdf72d4213d1c87d58"},"cell_type":"markdown","source":"# Load libs"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport datetime\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\nimport catboost as cb\n\nimport warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Load datasets"},{"metadata":{"trusted":true,"_uuid":"4c3cd402ac491b700733b5d4f8aadb28fe280997"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', parse_dates=[\"first_active_month\"])\ntest = pd.read_csv('../input/test.csv', parse_dates=[\"first_active_month\"])\n\nmerchants = pd.read_csv('../input/merchants.csv')\nhistorical_transactions = pd.read_csv('../input/historical_transactions.csv')\nnew_merchant_transactions = pd.read_csv('../input/new_merchant_transactions.csv')\n\nsample_submission = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0362ca65152509375fd49a22d325a88911c0454"},"cell_type":"code","source":"train.shape, test.shape, sample_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4e15aae66332aa5e5c14ab4f865b81ddea48de1"},"cell_type":"code","source":"merchants.shape, historical_transactions.shape, new_merchant_transactions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90674681af05480f168c7beba0478a8cbb90c1ef"},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb4a47b214b03fcd16e0aba3c7c1678d596758f0"},"cell_type":"code","source":"merchants.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f407d424663b8c52c2001a9c46269e9c4a7a9542"},"cell_type":"code","source":"historical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"798bc3747ef95fded87974f20f17fe70d7261131"},"cell_type":"code","source":"new_merchant_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"154ee07d62d69b05f848e6f764a71511551f551a"},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"trusted":true,"_uuid":"af2bd7d0cd507042bde869a5684a3bf4f29e2250"},"cell_type":"code","source":"def missing_impute(df):\n    for i in df.columns:\n        if df[i].dtype == \"object\":\n            df[i] = df[i].fillna(\"other\")\n        elif (df[i].dtype == \"int64\" or df[i].dtype == \"float64\"):\n            df[i] = df[i].fillna(df[i].mean())\n        else:\n            pass\n    return df\n\ndef datetime_extract(df, dt_col='first_active_month'):\n    df['date'] = df[dt_col].dt.date \n    df['day'] = df[dt_col].dt.day \n    df['dayofweek'] = df[dt_col].dt.dayofweek\n    df['dayofyear'] = df[dt_col].dt.dayofyear\n    df['days_in_month'] = df[dt_col].dt.days_in_month\n    df['daysinmonth'] = df[dt_col].dt.daysinmonth \n    df['month'] = df[dt_col].dt.month\n    df['week'] = df[dt_col].dt.week \n    df['weekday'] = df[dt_col].dt.weekday\n    df['weekofyear'] = df[dt_col].dt.weekofyear\n    df['year'] = train[dt_col].dt.year\n    \n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['date']).dt.days\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8a7ad122f650b831cdfe0c8fe134955b186afa2"},"cell_type":"code","source":"# Do impute missing values for all datasets\nfor df in [train, test, merchants, historical_transactions, new_merchant_transactions]:\n    missing_impute(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a030352928cbafe56dd1bb38c407c10f64bf1cb9"},"cell_type":"code","source":"# Do extract datetime values for train and test\ntrain = datetime_extract(train, dt_col='first_active_month')\ntest = datetime_extract(test, dt_col='first_active_month')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86d78656d604491948ffc3b0d818bc2d79d04726"},"cell_type":"markdown","source":"## Merge with historical transactions"},{"metadata":{"trusted":true,"_uuid":"97c1cc4c3b50dd07f090703553e9b3b4e0cdef43"},"cell_type":"code","source":"# Define the aggregation procedure outside of the groupby operation\naggregations = {\n    'purchase_amount': ['sum', 'mean', 'std', 'min', 'max', 'size', 'median']\n}\n\ngrouped = historical_transactions.groupby('card_id').agg(aggregations)\ngrouped.columns = grouped.columns.droplevel(level=0)\ngrouped.rename(columns={\n    \"sum\": \"sum_purchase_amount\", \n    \"mean\": \"mean_purchase_amount\",\n    \"std\": \"std_purchase_amount\", \n    \"min\": \"min_purchase_amount\",\n    \"max\": \"max_purchase_amount\", \n    \"size\": \"num_purchase_amount\",\n    \"median\": \"median_purchase_amount\"\n}, inplace=True)\ngrouped.reset_index(inplace=True)\n\ntrain = pd.merge(train, grouped, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, grouped, on=\"card_id\", how=\"left\")\n\ndel grouped\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ddf5724df8eefc1769f5e17d682b9b74a364e2d"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"918639a2c65f8d956f265d8e3d1988acea09a292"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"550c774001e99290735df52e4963adb153eeda2e"},"cell_type":"markdown","source":"## Merge with new merchant"},{"metadata":{"trusted":true,"_uuid":"aec35ef0039559e9dfc56f7e63ef2e6a434e73d6"},"cell_type":"code","source":"# Define the aggregation procedure outside of the groupby operation\naggregations = {\n    'purchase_amount': ['sum', 'mean', 'std', 'min', 'max', 'size', 'median']\n}\n\ngrouped = new_merchant_transactions.groupby('card_id').agg(aggregations)\ngrouped.columns = grouped.columns.droplevel(level=0)\ngrouped.rename(columns={\n    \"sum\": \"sum_purchase_amount\", \n    \"mean\": \"mean_purchase_amount\",\n    \"std\": \"std_purchase_amount\", \n    \"min\": \"min_purchase_amount\",\n    \"max\": \"max_purchase_amount\", \n    \"size\": \"num_purchase_amount\",\n    \"median\": \"median_purchase_amount\"\n}, inplace=True)\ngrouped.reset_index(inplace=True)\n\ntrain = pd.merge(train, grouped, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, grouped, on=\"card_id\", how=\"left\")\n\ndel grouped\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b7be9d8d3ca4f1ef1fb1938d854849321847505"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd1004a2df3bc22da850469370741c8f89e01443"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67ffb9b605fa4bb037f98338851f46587cefe967"},"cell_type":"markdown","source":"# One Hot Encoding"},{"metadata":{"trusted":true,"_uuid":"712c876f49a046ad29e806d6c8e86eb6e5c1202c"},"cell_type":"code","source":"# One-hot encode features\nohe_df_1 = pd.get_dummies(train['feature_1'], prefix='f1_')\nohe_df_2 = pd.get_dummies(train['feature_2'], prefix='f2_')\nohe_df_3 = pd.get_dummies(train['feature_3'], prefix='f3_')\n\nohe_df_4 = pd.get_dummies(test['feature_1'], prefix='f1_')\nohe_df_5 = pd.get_dummies(test['feature_2'], prefix='f2_')\nohe_df_6 = pd.get_dummies(test['feature_3'], prefix='f3_')\n\n# Numerical representation of the first active month\ntrain = pd.concat([train, ohe_df_1, ohe_df_2, ohe_df_3], axis=1, sort=False)\ntest = pd.concat([test, ohe_df_4, ohe_df_5, ohe_df_6], axis=1, sort=False)\n\ndel ohe_df_1, ohe_df_2, ohe_df_3\ndel ohe_df_4, ohe_df_5, ohe_df_6\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9890564517b610139b055b4947d4392fabe8027f"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb7d2be3f62be9b14fe14919b60b37585ef09335"},"cell_type":"code","source":"excluded_features = ['first_active_month', 'card_id', 'target', 'date', 'year']\ntrain_features = [c for c in train.columns if c not in excluded_features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f43ee69c7fb0ef5cf2ae7feccaa5fb7d128f29b5"},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a86c116a3e383272abc817fd61b20db3410f85f8"},"cell_type":"markdown","source":"--> Still missing values. So need to fill missing values again"},{"metadata":{"trusted":true,"_uuid":"0817dc04834e02cee91f766f0f3e02f9cb1e3199"},"cell_type":"code","source":"for col in train_features:\n    for df in [train, test]:\n        if df[col].dtype == \"float64\":\n            df[col] = df[col].fillna(df[col].mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5124e5825881a2e31d2b1f352cfb22a370124fc"},"cell_type":"markdown","source":"# CatBoost training"},{"metadata":{"trusted":true,"_uuid":"5da76845cce1ae5a6470a3efd0f8cccb2c4525e3"},"cell_type":"code","source":"# Prepare data for training\nX = train.copy()\ny = X['target']\n\n# Split data with kfold\nkfolds = KFold(n_splits=5, shuffle=True, random_state=2018)\n\n# Make importance dataframe\nimportances = pd.DataFrame()\n\noof_preds = np.zeros(X.shape[0])\nsub_preds = np.zeros(test.shape[0])\n\nfor n_fold, (trn_idx, val_idx) in enumerate(kfolds.split(X, y)):\n    X_train, y_train = X[train_features].iloc[trn_idx], y.iloc[trn_idx]\n    X_valid, y_valid = X[train_features].iloc[val_idx], y.iloc[val_idx]\n    \n    # CatBoost Regressor estimator\n    model = cb.CatBoostRegressor(\n        learning_rate = 0.03,\n        iterations = 1000,\n        eval_metric = 'RMSE',\n        allow_writing_files = False,\n        od_type = 'Iter',\n        bagging_temperature = 0.2,\n        depth = 10,\n        od_wait = 20,\n        silent = True\n    )\n    \n    # Fit\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_train, y_train), (X_valid, y_valid)],\n        verbose=None,\n        early_stopping_rounds=100\n    )\n    \n    # Feature importance\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = model.get_feature_importance()\n    imp_df['fold'] = n_fold + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_preds[val_idx] = model.predict(X_valid)\n    test_preds = model.predict(test[train_features])\n    sub_preds += test_preds / kfolds.n_splits\n    \nprint(np.sqrt(mean_squared_error(y, oof_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae01d50140e93d67512b8a44e12eb31b4d7b39ab"},"cell_type":"markdown","source":"# Feature Importances"},{"metadata":{"trusted":true,"_uuid":"13eb67fea441124c4daccacbe6c575e327d757ce"},"cell_type":"code","source":"importances['gain_log'] = importances['gain']\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"028de7d83e90c79bca81e5c1aafc02c4a2fddd54"},"cell_type":"markdown","source":"# Make submission"},{"metadata":{"trusted":true,"_uuid":"088409ecfc31836a4a3617097ef321fa6e8cc9c4"},"cell_type":"code","source":"# Length of submission\nlen(sub_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7d0802e0492078e10331daca63b87c7988c511e"},"cell_type":"code","source":"# Make submission\nsample_submission['target'] = sub_preds\nsample_submission.to_csv(\"cb_submission.csv\", index=False)\nsample_submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}