{"cells":[{"metadata":{"_uuid":"0ed6b23b8871d677e707ef18d284d72e1fb60430"},"cell_type":"markdown","source":"# Notes\n\nThis folk from [https://www.kaggle.com/truocpham/feature-engineering-and-lightgbm-starter](https://www.kaggle.com/truocpham/feature-engineering-and-lightgbm-starter) (thanks Truoc) and makes some changes:\n* Remove the **year** feature\n* Update the params for the lgb model"},{"metadata":{"_uuid":"9359af78b09b2e265605e9360818c6759989ee15"},"cell_type":"markdown","source":"# Load packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\nimport lightgbm as lgb\n\nimport warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32ffc0efe8909024eaf75cb98bc60cb37e36e4bb"},"cell_type":"markdown","source":"# Load datasets"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', parse_dates=[\"first_active_month\"])\ntest = pd.read_csv('../input/test.csv', parse_dates=[\"first_active_month\"])\nsample_submission = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1201bd18a40de89e0d8d9780d0d0826d6b71fc94"},"cell_type":"code","source":"train.shape, test.shape, sample_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00367965ddadab0f105bbd77ed9c739580a6875a"},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0544675ceed4ad3e38c990c98dc8305bfac0bf4"},"cell_type":"code","source":"test.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"776ffcc1c03d9ab226c9861ba3c775670705f88c"},"cell_type":"code","source":"merchants = pd.read_csv('../input/merchants.csv')\nhistorical_transactions = pd.read_csv('../input/historical_transactions.csv')\nnew_merchant_transactions = pd.read_csv('../input/new_merchant_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5bd3cb9e4e5b689352c6bb862d53b34e70cd512"},"cell_type":"code","source":"merchants.shape, historical_transactions.shape, new_merchant_transactions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a099d8aac57f289ad376c6f1e3d96cf9f3e6a49a"},"cell_type":"code","source":"merchants.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73b1f3a2fd71074061172703e0cc9338e81199fe"},"cell_type":"code","source":"historical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df1b42bb42c4e1ab833fe229e32767e48034f70b"},"cell_type":"code","source":"new_merchant_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06afee6fae5e96af10b316296d539f3afb8094c3"},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"trusted":true,"_uuid":"5690bebe7b59d7f6b6acfe7836e82d45b65974fc"},"cell_type":"code","source":"def missing_impute(df):\n    for i in df.columns:\n        if df[i].dtype == \"object\":\n            df[i] = df[i].fillna(\"other\")\n        elif (df[i].dtype == \"int64\" or df[i].dtype == \"float64\"):\n            df[i] = df[i].fillna(df[i].mean())\n        else:\n            pass\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab18ef780c79de30abaff5937d2706605bbabc60"},"cell_type":"code","source":"def datetime_extract(df, dt_col='first_active_month'):\n    df['date'] = df[dt_col].dt.date \n    df['day'] = df[dt_col].dt.day \n    df['dayofweek'] = df[dt_col].dt.dayofweek\n    df['dayofyear'] = df[dt_col].dt.dayofyear\n    df['days_in_month'] = df[dt_col].dt.days_in_month\n    df['daysinmonth'] = df[dt_col].dt.daysinmonth \n    df['month'] = df[dt_col].dt.month\n    df['week'] = df[dt_col].dt.week \n    df['weekday'] = df[dt_col].dt.weekday\n    df['weekofyear'] = df[dt_col].dt.weekofyear\n#     df['year'] = train[dt_col].dt.year\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6aee2a3c0f89f20fe050fa28e429c70321876e0"},"cell_type":"code","source":"# Do impute missing values\nfor df in [train, test, merchants, historical_transactions, new_merchant_transactions]:\n    missing_impute(df)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"f5fb2ea6c9dead11a98c5579370ec895834568e6"},"cell_type":"code","source":"# Do extract datetime values\ntrain = datetime_extract(train, dt_col='first_active_month')\ntest = datetime_extract(test, dt_col='first_active_month')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbe41130a6ecf72dbe6fa001c8cbb44529f7a86d"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0972e3086726b641bb2b61cdfd8c639f978abc1"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70bf49e0fa2a09df07843b06bd53b5600077511d"},"cell_type":"markdown","source":"**Merge train and test with historical transactions**"},{"metadata":{"trusted":true,"_uuid":"dfc05969b76d08f7a9a231aae74b0ddb9dc8ee1a"},"cell_type":"code","source":"# Define the aggregation procedure outside of the groupby operation\naggregations = {\n    'purchase_amount': ['sum', 'mean', 'std', 'min', 'max', 'size'],\n}\n\ngrouped = historical_transactions.groupby('card_id').agg(aggregations)\ngrouped.columns = grouped.columns.droplevel(level=0)\ngrouped.rename(columns={\n    \"sum\": \"sum_purchase_amount\", \n    \"mean\": \"mean_purchase_amount\",\n    \"std\": \"std_purchase_amount\", \n    \"min\": \"min_purchase_amount\",\n    \"max\": \"max_purchase_amount\", \n    \"size\": \"num_purchase_amount\"}, inplace=True)\ngrouped.reset_index(inplace=True)\n\ntrain = pd.merge(train, grouped, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, grouped, on=\"card_id\", how=\"left\")\n\ndel grouped\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c0a4aa17d4d83a6ca7f029e99b42e8be778d255"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e604a039111fc20210812ffdee18a1a880fcd6c6"},"cell_type":"markdown","source":"**Merge train and test with new_merchant_transactions**"},{"metadata":{"trusted":true,"_uuid":"6e687ad952f1e297497b7b37dbca51cad3c37bf7"},"cell_type":"code","source":"# Define the aggregation procedure outside of the groupby operation\naggregations = {\n    'purchase_amount': ['sum', 'mean', 'std', 'min', 'max', 'size'],\n}\n\ngrouped = new_merchant_transactions.groupby('card_id').agg(aggregations)\ngrouped.columns = grouped.columns.droplevel(level=0)\ngrouped.rename(columns={\n    \"sum\": \"sum_purchase_amount\", \n    \"mean\": \"mean_purchase_amount\",\n    \"std\": \"std_purchase_amount\", \n    \"min\": \"min_purchase_amount\",\n    \"max\": \"max_purchase_amount\", \n    \"size\": \"num_purchase_amount\"}, inplace=True)\ngrouped.reset_index(inplace=True)\n\ntrain = pd.merge(train, grouped, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, grouped, on=\"card_id\", how=\"left\")\n\ndel grouped\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d18e97e6da48e367179672b4d377aa95071cb07"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2985c37c157f13a2e1be4778dbd4173635f0a2d3"},"cell_type":"markdown","source":"# Featuring"},{"metadata":{"trusted":true,"_uuid":"a09655e2590ffb3d5a44d11244f4d9a6ab551280"},"cell_type":"code","source":"# One-hot encode features\nohe_df_1 = pd.get_dummies(train['feature_1'], prefix='f1_')\nohe_df_2 = pd.get_dummies(train['feature_2'], prefix='f2_')\nohe_df_3 = pd.get_dummies(train['feature_3'], prefix='f3_')\n\nohe_df_4 = pd.get_dummies(test['feature_1'], prefix='f1_')\nohe_df_5 = pd.get_dummies(test['feature_2'], prefix='f2_')\nohe_df_6 = pd.get_dummies(test['feature_3'], prefix='f3_')\n\n# Numerical representation of the first active month\ntrain = pd.concat([train, ohe_df_1, ohe_df_2, ohe_df_3], axis=1, sort=False)\ntest = pd.concat([test, ohe_df_4, ohe_df_5, ohe_df_6], axis=1, sort=False)\n\ndel ohe_df_1, ohe_df_2, ohe_df_3\ndel ohe_df_4, ohe_df_5, ohe_df_6\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6e8f84df895d6a4765920167f82806417e12ebd"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da481b145fed2f9772d3067dbace13249b5e9a43"},"cell_type":"code","source":"excluded_features = ['first_active_month', 'card_id', 'target', 'date']\ntrain_features = [c for c in train.columns if c not in excluded_features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf2d88fdd55aec44c2fdf17cb126f7e963a3a702"},"cell_type":"code","source":"for f in train_features:\n    print(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9debf2ef17da0e7adf878746a6bde6141a847eb4"},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d424bc5608f861f6dd37cce03bb2404a81c14d31"},"cell_type":"markdown","source":"--> Still missing values. So need to fill NA again"},{"metadata":{"trusted":true,"_uuid":"fa8b31197f0463bb91ee1061f00fdc17a0e83b28"},"cell_type":"code","source":"for col in train_features:\n    for df in [train, test]:\n        if df[col].dtype == \"float64\":\n            df[col] = df[col].fillna(df[col].mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3eb1aa7918ca54128f8fb03a441c79ffcf3f9040"},"cell_type":"markdown","source":"# Modeling with LightGBM"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"89ea69cadb4fe54b9976aec33a83e2584d9d0a88"},"cell_type":"code","source":"# Prepare data for training\nX = train.copy()\ny = X['target']\n\n# Split data with kfold\nkfolds = KFold(n_splits=5, shuffle=True, random_state=2018)\n\n# Make importance dataframe\nimportances = pd.DataFrame()\n\noof_preds = np.zeros(X.shape[0])\nsub_preds = np.zeros(test.shape[0])\n\nfor n_fold, (trn_idx, val_idx) in enumerate(kfolds.split(X, y)):\n    X_train, y_train = X[train_features].iloc[trn_idx], y.iloc[trn_idx]\n    X_valid, y_valid = X[train_features].iloc[val_idx], y.iloc[val_idx]\n    \n    # LightGBM Regressor estimator\n    model = lgb.LGBMRegressor(\n        num_leaves = 31,\n        learning_rate = 0.03,\n        n_estimators = 1000,\n        subsample = .9,\n        colsample_bytree = .9,\n        random_state = 1,\n        booster = \"gbtree\",\n        eval_metric = \"rmse\",\n        nthread = 4,\n        nrounds = 1500,\n        max_depth = 7\n    )\n    \n    # Fit\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_train, y_train), (X_valid, y_valid)],\n        verbose=None, eval_metric='rmse',\n        early_stopping_rounds=100\n    )\n    \n    # Feature importance\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = model.booster_.feature_importance(importance_type='gain')\n    imp_df['fold'] = n_fold + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_preds[val_idx] = model.predict(X_valid, num_iteration=model.best_iteration_)\n    test_preds = model.predict(test[train_features], num_iteration=model.best_iteration_)\n    sub_preds += test_preds / kfolds.n_splits\n    \nmean_squared_error(y, oof_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75c915da1194cf72e6a780c75a665e9821a6370c"},"cell_type":"markdown","source":"# Display feature importances"},{"metadata":{"trusted":true,"_uuid":"d1c10c5f7153f931b123307f487744f4e0b6e947"},"cell_type":"code","source":"importances['gain_log'] = importances['gain']\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91d1dce4cf5fc9f88bf93eb6633ca8c540b100bb"},"cell_type":"markdown","source":"# Make submission"},{"metadata":{"trusted":true,"_uuid":"f88dc79e4277e9e20d15a8e5ad385edd7faf8ca0"},"cell_type":"code","source":"sub_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"110346855c0f867d9d060dc415f18d32ff68d0b8"},"cell_type":"code","source":"# Make submission 1\nsample_submission['target'] = sub_preds\nsample_submission.to_csv(\"submission1.csv\", index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"146746a9a9f08a73586ce62f7087c7685a7deaa0"},"cell_type":"code","source":"# Make submission 2 - revert from submission 1\nsample_submission['target'] = -1*sub_preds\nsample_submission.to_csv(\"submission2.csv\", index=False)\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16f3568e5dd4758a45213fc751682d7a509ce1a3"},"cell_type":"code","source":"# Length of submission\nlen(sub_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d4139ef8c96633c435d3d0469e38413b4d8bf6f"},"cell_type":"code","source":"# How many positive target values\nnp.sum(np.array(sub_preds) >= 0, axis=0)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}