{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nimport warnings\nimport time\nimport sys\nimport datetime\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nwarnings.simplefilter(action='ignore', category=FutureWarning)\npd.set_option('display.max_columns', 500)\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\ntest_df = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])\nprint(\"Number of rows and columns in train set : \",train_df.shape)\nprint(\"Number of rows and columns in test set : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3be38ae7c5e38bff72be7b9ff5a89e9a7181963"},"cell_type":"code","source":"train_df['feature_1'] = train_df['feature_1'].astype('category')\ntrain_df['feature_2'] = train_df['feature_2'].astype('category')\ntrain_df['feature_3'] = train_df['feature_3'].astype('category')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"defc6a6c69bc55b4c1da323f6123da882778c480"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize = (16, 6))\nplt.suptitle('Violineplots for features and target');\nsns.violinplot(x=\"feature_1\", y=\"target\", data=train_df, ax=ax[0], title='feature_1');\nsns.violinplot(x=\"feature_2\", y=\"target\", data=train_df, ax=ax[1], title='feature_2');\nsns.violinplot(x=\"feature_3\", y=\"target\", data=train_df, ax=ax[2], title='feature_3');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f796b9094546c0c47a75b87848962beb53a3a4ec"},"cell_type":"code","source":"f, ax = plt.subplots(figsize=(14, 6))\nsns.distplot(train_df['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cc5b2fe14738e9a3c6dd1e5392c72c5835263c3"},"cell_type":"code","source":"train_df['target'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a95635bde95f0b1efa5b592f0ab55320518f4b99"},"cell_type":"code","source":"fig, ((ax1, ax2)) = plt.subplots(nrows=1, ncols=2, figsize=(14,8))\ndata = pd.concat([train_df['target'], train_df['feature_1']], axis=1)\nfig = sns.boxplot(x='feature_1', y=\"target\", data=data,ax=ax1)\nsns.violinplot(x=\"feature_1\", y=\"target\", data=data,ax=ax2)\nplt.xticks(rotation='vertical')\nplt.xlabel('Feature 1', fontsize=12)\nplt.ylabel('target', fontsize=12)\nplt.title(\"Feature 1 distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f4db2900cf1b1c185b20b013ca0117ea6ea846c"},"cell_type":"code","source":"fig, ((ax1, ax2)) = plt.subplots(nrows=1, ncols=2, figsize=(14,8))\n\ndata = pd.concat([train_df['target'], train_df['feature_3']], axis=1)\nfig = sns.boxplot(x='feature_3', y=\"target\", data=data,ax=ax1)\n\n# feature 1\nsns.violinplot(x=\"feature_3\", y=\"target\", data=data,ax=ax2)\nplt.xticks(rotation='vertical')\nplt.xlabel('feature_3', fontsize=12)\nplt.ylabel('target', fontsize=12)\nplt.title(\"feature_3 distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1836dbacc8b1421c1d931dbec6f79c3edf8bd096"},"cell_type":"code","source":"fig, ((ax1, ax2)) = plt.subplots(nrows=1, ncols=2, figsize=(14,8))\n\nsns.boxplot(x=\"feature_1\", y=\"target\", hue=\"feature_3\",\n               data=train_df, palette=\"Set3\",ax=ax1)\n\nsns.boxplot(x=\"feature_3\", y=\"target\", hue=\"feature_1\",\n               data=train_df, palette=\"Set3\",ax=ax2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f732d4eb6aa30ed09601320869ab89706449412d"},"cell_type":"code","source":"fig, ((ax1, ax2)) = plt.subplots(nrows=1, ncols=2, figsize=(14,8))\n\nsns.boxplot(x=\"feature_1\", y=\"target\", hue=\"feature_2\",\n               data=train_df, palette=\"Set3\",ax=ax1)\n\nsns.boxplot(x=\"feature_2\", y=\"target\", hue=\"feature_1\",\n               data=train_df, palette=\"Set3\",ax=ax2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab74fd39d5633392954283dcb0108de15b8f0e30"},"cell_type":"code","source":"def missing_data(train_df):\n    total = data.isnull().sum().sort_values(ascending = False)\n    percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)\n    return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3eb0276546850a46021ee16978b5983cb9d8f032"},"cell_type":"code","source":"missing_data(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4da62b5f2a35954a1d230d71c2bfa255d452fda1"},"cell_type":"code","source":"import datetime\n\nfor df in [train_df,test_df]:\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['year'] = df['first_active_month'].dt.year\n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n\ntarget = train_df['target']\ndel train_df['target']\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9e4c0649c30df92f26ae351ae303ca68f4a769d"},"cell_type":"code","source":"hist_df = pd.read_csv(\"../input/historical_transactions.csv\")\nprint(\"shape of historical_transactions : \",hist_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54e44eba133a43cd41a9659c68bc2538d2c04300"},"cell_type":"code","source":"temp = hist_df[\"category_1\"].value_counts()\ndf = pd.DataFrame({'labels': temp.index,\n                   'values': temp.values\n                  })\nplt.figure(figsize = (6,6))\nplt.title('category_1 - Y or N')\nsns.set_color_codes(\"pastel\")\nsns.barplot(x = 'labels', y=\"values\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bb775ffd8bd7fa755018cb8b11d6e6f5a1abb1e"},"cell_type":"code","source":"temp = hist_df[\"category_3\"].value_counts()\ndf = pd.DataFrame({'labels': temp.index,\n                   'values': temp.values\n                  })\nplt.figure(figsize = (6,6))\nplt.title('category_3 - A B C')\nsns.set_color_codes(\"pastel\")\nsns.barplot(x = 'labels', y=\"values\", data=df)\nlocs, labels = plt.xticks()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d214b790a946110381a4accc4bb0dfd1299a867"},"cell_type":"code","source":"temp = hist_df[\"category_3\"].value_counts()\ndf = pd.DataFrame({'labels': temp.index,\n                   'values': temp.values\n                  })\nplt.figure(figsize = (6,6))\nplt.title('category_2 - 1,2,3,4,5')\nsns.set_color_codes(\"pastel\")\nsns.barplot(x = 'labels', y=\"values\", data=df)\nlocs, labels = plt.xticks()\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7da55da2ff90c1a35e1d55be532f4f40a39f5e0"},"cell_type":"code","source":"f, ax = plt.subplots(figsize=(15, 8))\nsns.distplot(hist_df['month_lag'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"581ec4d4a6b01cd13f93456d6d0c74fef8e837fb"},"cell_type":"code","source":"use_cols = [col for col in train_df.columns if col not in ['card_id', 'first_active_month']]\n\ntrain = train_df[use_cols]\ntest = test_df[use_cols]\n\nfeatures = list(train_df[use_cols].columns)\ncategorical_feats = [col for col in features if 'feature_' in col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"850664d7bd1c8f94acfe2f5314cc3ba4219b6348"},"cell_type":"code","source":"for col in categorical_feats:\n    print(col, 'have', train_df[col].value_counts().shape[0], 'categories.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7dd2c0c6ff2516548cbe5dbc8728ad08d197fd2a"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfor col in categorical_feats:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n    train[col] = lbl.transform(list(train[col].values.astype('str')))\n    test[col] = lbl.transform(list(test[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4ea818c7aec7df1626bccdf46979cd33eee1877"},"cell_type":"code","source":"df_all = pd.concat([train, test])\ndf_all = pd.get_dummies(df_all, columns=categorical_feats)\n\nlen_train = train.shape[0]\n\ntrain = df_all[:len_train]\ntest = df_all[len_train:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb36b0798c308f18c911c2c8ce2680be63799e2d"},"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb\n\n\nlgb_params = {\"objective\" : \"regression\", \"metric\" : \"rmse\", \n               \"max_depth\": 7, \"min_child_samples\": 20, \n               \"reg_alpha\": 1, \"reg_lambda\": 1,\n               \"num_leaves\" : 64, \"learning_rate\" : 0.005, \n               \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \n               \"verbosity\": -1}\n\nFOLDs = KFold(n_splits=5, shuffle=True, random_state=1989)\n\noof_lgb = np.zeros(len(train))\npredictions_lgb = np.zeros(len(test))\n\nfeatures_lgb = list(train.columns)\nfeature_importance_df_lgb = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = lgb.Dataset(train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx], label=target.iloc[val_idx])\n\n    print(\"LGB \" + str(fold_) + \"-\" * 50)\n    num_round = 10000\n    clf = lgb.train(lgb_params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 50)\n    oof_lgb[val_idx] = clf.predict(train.iloc[val_idx], num_iteration=clf.best_iteration)\n\n    fold_importance_df_lgb = pd.DataFrame()\n    fold_importance_df_lgb[\"feature\"] = features_lgb\n    fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n    fold_importance_df_lgb[\"fold\"] = fold_ + 1\n    feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n    predictions_lgb += clf.predict(test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n    \n\nprint(np.sqrt(mean_squared_error(oof_lgb, target)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"284cdb8fab75c3ae97fb526ae2ae9c9bd45a0444"},"cell_type":"code","source":"cols = (feature_importance_df_lgb[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\n\nplt.figure(figsize=(14,14))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1bbb255ec976a69eb521a65a9cdf13c0e5ee7df"},"cell_type":"code","source":"sub_df = pd.read_csv('../input/sample_submission.csv')\nsub_df[\"target\"] = predictions_lgb \nsub_df.to_csv(\"submission_lgb.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5d66908402d624ae9b426e04caf21dc50ef1f74"},"cell_type":"code","source":"import xgboost as xgb\n\nxgb_params = {'eta': 0.005, 'max_depth': 7, 'subsample': 0.8, 'colsample_bytree': 0.8, \n          'objective': 'reg:linear', 'eval_metric': 'rmse', 'silent': True}\n\n\n\nFOLDs = KFold(n_splits=5, shuffle=True, random_state=1989)\n\noof_xgb = np.zeros(len(train))\npredictions_xgb = np.zeros(len(test))\n\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = xgb.DMatrix(data=train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = xgb.DMatrix(data=train.iloc[val_idx], label=target.iloc[val_idx])\n    watchlist = [(trn_data, 'train'), (val_data, 'valid')]\n    print(\"xgb \" + str(fold_) + \"-\" * 50)\n    num_round = 10000\n    xgb_model = xgb.train(xgb_params, trn_data, num_round, watchlist, early_stopping_rounds=50, verbose_eval=1000)\n    oof_xgb[val_idx] = xgb_model.predict(xgb.DMatrix(train.iloc[val_idx]), ntree_limit=xgb_model.best_ntree_limit+50)\n\n    predictions_xgb += xgb_model.predict(xgb.DMatrix(test), ntree_limit=xgb_model.best_ntree_limit+50) / FOLDs.n_splits\n\nnp.sqrt(mean_squared_error(oof_xgb, target))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f54ab4d1349642cfb4a50e6e042951cbc6aa4af"},"cell_type":"code","source":"print('lgb', np.sqrt(mean_squared_error(oof_lgb, target)))\nprint('xgb', np.sqrt(mean_squared_error(oof_xgb, target)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c7150a038ed777a56feec15c51689a26212c0a1"},"cell_type":"code","source":"total_sum = 0.5 * oof_lgb + 0.5 * oof_xgb\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(total_sum, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61cf6784d941d207aa35578f1d049eb19313f09c"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test_df[\"card_id\"].values})\nsub_df[\"target\"] = 0.5 * predictions_lgb + 0.5 * predictions_xgb\nsub_df.to_csv(\"submission_ensemble.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}