{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Notes\n\n* This kernel is cover baseline modeling with XGBoost"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Load libs"},{"metadata":{"trusted":true,"_uuid":"8e0017da876cc8201f29b9b9f119ebeca0869a5b"},"cell_type":"code","source":"import os\nimport gc\nimport datetime\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\nimport xgboost as xgb\n\nimport warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a323005a6e342b845a861b613c4912cece5f52a3"},"cell_type":"markdown","source":"# Load datasets"},{"metadata":{"trusted":true,"_uuid":"ee136243db6e978137088a70ab57a52fa9ba2a1e"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', parse_dates=[\"first_active_month\"])\ntest = pd.read_csv('../input/test.csv', parse_dates=[\"first_active_month\"])\n\nmerchants = pd.read_csv('../input/merchants.csv')\nhistorical_transactions = pd.read_csv('../input/historical_transactions.csv')\nnew_merchant_transactions = pd.read_csv('../input/new_merchant_transactions.csv')\n\nsample_submission = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1d83fdbe2d89c5e85769721f69e3a586174015e"},"cell_type":"code","source":"train.shape, test.shape, sample_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f53302a6c5b262fe68408d2c10f793f7a89a80de"},"cell_type":"code","source":"merchants.shape, historical_transactions.shape, new_merchant_transactions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d7abaebe1b8730f6c2749f16feb5866a71ac25f"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eabf3e4619465c0c6b3705e5010b944b4e5cbdf5"},"cell_type":"code","source":"merchants.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14043583eb7bec3df5fae75863d0f81a526fbf2f"},"cell_type":"code","source":"historical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59d5045eabe7d134f04072dcb00d76892abea476"},"cell_type":"code","source":"new_merchant_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"47a4a48863aaacaf4d19572d8bba08522378265b"},"cell_type":"markdown","source":"# Preprocessing\n"},{"metadata":{"trusted":true,"_uuid":"df59e0cf9c1751b6aef2d34a4dfd8eed6af7782d"},"cell_type":"code","source":"def missing_impute(df):\n    for i in df.columns:\n        if df[i].dtype == \"object\":\n            df[i] = df[i].fillna(\"other\")\n        elif (df[i].dtype == \"int64\" or df[i].dtype == \"float64\"):\n            df[i] = df[i].fillna(df[i].mean())\n        else:\n            pass\n    return df\n\ndef datetime_extract(df, dt_col='first_active_month'):\n    df['date'] = df[dt_col].dt.date \n    df['day'] = df[dt_col].dt.day \n    df['dayofweek'] = df[dt_col].dt.dayofweek\n    df['dayofyear'] = df[dt_col].dt.dayofyear\n    df['days_in_month'] = df[dt_col].dt.days_in_month\n    df['daysinmonth'] = df[dt_col].dt.daysinmonth \n    df['month'] = df[dt_col].dt.month\n    df['week'] = df[dt_col].dt.week \n    df['weekday'] = df[dt_col].dt.weekday\n    df['weekofyear'] = df[dt_col].dt.weekofyear\n    df['year'] = train[dt_col].dt.year\n    \n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['date']).dt.days\n\n    return df\n\n\n# Do impute missing values for all datasets\nfor df in [train, test, merchants, historical_transactions, new_merchant_transactions]:\n    missing_impute(df)\n    \n\n# Do extract datetime values for train and test\ntrain = datetime_extract(train, dt_col='first_active_month')\ntest = datetime_extract(test, dt_col='first_active_month')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8da7dadda8a9866aa3f2d99aceb3fbc4b4f9235a"},"cell_type":"markdown","source":"## Merge with historical transactions"},{"metadata":{"trusted":true,"_uuid":"56338c0b0c6331bc50c107a200d959fb7688c7a5"},"cell_type":"code","source":"# Define the aggregation procedure outside of the groupby operation\naggregations = {\n    'purchase_amount': ['sum', 'mean', 'std', 'min', 'max', 'size', 'median']\n}\n\ngrouped = historical_transactions.groupby('card_id').agg(aggregations)\ngrouped.columns = grouped.columns.droplevel(level=0)\ngrouped.rename(columns={\n    \"sum\": \"sum_purchase_amount\", \n    \"mean\": \"mean_purchase_amount\",\n    \"std\": \"std_purchase_amount\", \n    \"min\": \"min_purchase_amount\",\n    \"max\": \"max_purchase_amount\", \n    \"size\": \"num_purchase_amount\",\n    \"median\": \"median_purchase_amount\"\n}, inplace=True)\ngrouped.reset_index(inplace=True)\n\ntrain = pd.merge(train, grouped, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, grouped, on=\"card_id\", how=\"left\")\n\ndel grouped\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8c3ab3351e17f980bde6d46032a212324f15367"},"cell_type":"markdown","source":"## Merge with new merchant"},{"metadata":{"trusted":true,"_uuid":"bece9ea9110bc11cfff046e28c06f6c6c1db4ea6"},"cell_type":"code","source":"# Define the aggregation procedure outside of the groupby operation\naggregations = {\n    'purchase_amount': ['sum', 'mean', 'std', 'min', 'max', 'size', 'median']\n}\n\ngrouped = new_merchant_transactions.groupby('card_id').agg(aggregations)\ngrouped.columns = grouped.columns.droplevel(level=0)\ngrouped.rename(columns={\n    \"sum\": \"sum_purchase_amount\", \n    \"mean\": \"mean_purchase_amount\",\n    \"std\": \"std_purchase_amount\", \n    \"min\": \"min_purchase_amount\",\n    \"max\": \"max_purchase_amount\", \n    \"size\": \"num_purchase_amount\",\n    \"median\": \"median_purchase_amount\"\n}, inplace=True)\ngrouped.reset_index(inplace=True)\n\ntrain = pd.merge(train, grouped, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, grouped, on=\"card_id\", how=\"left\")\n\ndel grouped\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"507215e4c16922c909f0d03157fcd9702426aa9c"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fedd853f22f6f2fe45a324d252cee8a06612024"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"45c8a4b92a2411ad9ddf9294f618698d0e72fc23"},"cell_type":"markdown","source":"# One Hot Encoding"},{"metadata":{"trusted":true,"_uuid":"009569faf531970cb2e18c9b45cc1e8e9aba8103"},"cell_type":"code","source":"# One-hot encode features\nohe_df_1 = pd.get_dummies(train['feature_1'], prefix='f1_')\nohe_df_2 = pd.get_dummies(train['feature_2'], prefix='f2_')\nohe_df_3 = pd.get_dummies(train['feature_3'], prefix='f3_')\n\nohe_df_4 = pd.get_dummies(test['feature_1'], prefix='f1_')\nohe_df_5 = pd.get_dummies(test['feature_2'], prefix='f2_')\nohe_df_6 = pd.get_dummies(test['feature_3'], prefix='f3_')\n\n# Numerical representation of the first active month\ntrain = pd.concat([train, ohe_df_1, ohe_df_2, ohe_df_3], axis=1, sort=False)\ntest = pd.concat([test, ohe_df_4, ohe_df_5, ohe_df_6], axis=1, sort=False)\n\ndel ohe_df_1, ohe_df_2, ohe_df_3\ndel ohe_df_4, ohe_df_5, ohe_df_6\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7577ab7777cb86900bfe88431435d5c681e74a6"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b825cb68ee21cd43f1cf4700bfa0b9a2f6c536a"},"cell_type":"code","source":"excluded_features = ['first_active_month', 'card_id', 'target', 'date', 'year']\ntrain_features = [c for c in train.columns if c not in excluded_features]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f434ba76da5cbdaf84ca2fb732c19630e0306fe7"},"cell_type":"markdown","source":"## Final fill NA"},{"metadata":{"trusted":true,"_uuid":"c70975f54276470914c4ec158b783c4c9dea0978"},"cell_type":"code","source":"# Final fill missing values\nfor col in train_features:\n    for df in [train, test]:\n        if df[col].dtype == \"float64\":\n            df[col] = df[col].fillna(df[col].mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f79be7a0111c18365f17c6784db1f88a52b109f"},"cell_type":"markdown","source":"# XGBoost training"},{"metadata":{"trusted":true,"_uuid":"d0537d21d601a6fb11f1ee22125dd4eb9c647c57"},"cell_type":"code","source":"# Prepare data for training\nX = train.copy()\ny = X['target']\n\n# Split data with kfold\nkfolds = KFold(n_splits=5, shuffle=True, random_state=2018)\n\n# Make importance dataframe\nimportances = pd.DataFrame()\n\noof_preds = np.zeros(X.shape[0])\nsub_preds = np.zeros(test.shape[0])\n\nfor n_fold, (trn_idx, val_idx) in enumerate(kfolds.split(X, y)):\n    X_train, y_train = X[train_features].iloc[trn_idx], y.iloc[trn_idx]\n    X_valid, y_valid = X[train_features].iloc[val_idx], y.iloc[val_idx]\n    \n    # XGBoost Regressor estimator\n    model = xgb.XGBRegressor(\n        max_depth = 31,\n        learning_rate = 0.03,\n        n_estimators = 1000,\n        subsample = .9,\n        colsample_bylevel = .9,\n        colsample_bytree = .9,\n        min_child_weight= .9,\n        gamma = 0,\n        random_state = 100,\n        booster = 'gbtree',\n        objective = 'reg:linear'\n    )\n    \n    # Fit\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_train, y_train), (X_valid, y_valid)],\n        verbose=None, eval_metric='rmse',\n        early_stopping_rounds=100\n    )\n    \n    # Feature importance\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = model.feature_importances_\n    imp_df['fold'] = n_fold + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_preds[val_idx] = model.predict(X_valid)\n    test_preds = model.predict(test[train_features])\n    sub_preds += test_preds / kfolds.n_splits\n    \nprint(np.sqrt(mean_squared_error(y, oof_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec1fab316578c135e4e2268bdaa1545cf8d6ee86"},"cell_type":"markdown","source":"# Feature Importances"},{"metadata":{"trusted":true,"_uuid":"18373a4a0d241ae8049957e8284d2eb2990f45c1"},"cell_type":"code","source":"importances['gain_log'] = importances['gain']\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9bb1182972b985c5f97277ec4732ba2970781613"},"cell_type":"markdown","source":"# Make submission"},{"metadata":{"trusted":true,"_uuid":"4c32c65e0c3b14ea3a24321fbee92630486eb3d5"},"cell_type":"code","source":"# Length of submission\nlen(sub_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3372ce5f58658b9ea90e8315f0431f3b6cc2b586"},"cell_type":"code","source":"sub_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"504bbbfd3ddc9ad45eec514b1e8fd9c57ba00485"},"cell_type":"code","source":"# Make submission\nsample_submission['target'] = sub_preds\nsample_submission.to_csv(\"xgb_submission.csv\", index=False)\nsample_submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}