{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport datetime\nimport matplotlib.pyplot as plt\nimport xgboost as xgb\nfrom xgboost.sklearn import XGBRegressor\nfrom sklearn.model_selection import GridSearchCV\nfrom pandas import DataFrame\nfrom scipy import stats\nfrom matplotlib.pylab import rcParams\nfrom sklearn import metrics\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a352f1dd32d50b89622d18d40ea168ae8443e86b"},"cell_type":"code","source":"## Reading files\n\ntrain = pd.read_csv(\"../input/elo-merchant-category-recommendation/train.csv\")\ntest = pd.read_csv(\"../input/elo-merchant-category-recommendation/test.csv\")\n# hist_txns = pd.read_csv(\"../input/historical_transactions.csv\")\n# new_txns = pd.read_csv(\"../input/new_merchant_transactions.csv\")\n# merchants = pd.read_csv(\"../input/elo-merchant-category-recommendation/merchants.csv\")\n# all_txns = pd.read_csv(\"../input/transactions-data/all_txns.csv\")\n# all_txns = pd.concat([hist_txns, new_txns], ignore_index=True)\n# all_txns.to_csv(\"all_txns.csv\", index=False)\ngrouped_txns = pd.read_pickle(\"../input/transactions-data-with-pivoted-category-columns/grouped_txns.pickle\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e85d2aed7612639c396449e320965e69bb9a8aab"},"cell_type":"code","source":"# txns_subset = all_txns[[\"card_id\", \"purchase_amount\", \"month_lag\",\n#                         \"installments\", \"authorized_flag\", \"category_1\", \"category_3\"]]\n# # print (txns_subset.head(50))\n# grouped_txns = txns_subset.groupby(\"card_id\").agg({\n#     \"purchase_amount\": [np.sum, np.mean],\n#     \"month_lag\": [np.sum, np.mean],\n#     \"installments\": [np.sum, np.mean],\n#     \"authorized_flag\": {\"Y\": lambda x: x[x == 'Y'].count(),\n#                         \"N\": lambda x: x[x == 'N'].count()},\n#     \"category_1\": {\"Y\": lambda x: x[x == 'Y'].count(),\n#                    \"N\": lambda x: x[x == 'N'].count()},\n#     \"category_3\": {\"A\": lambda x: x[x == 'A'].count(),\n#                    \"B\": lambda x: x[x == 'B'].count(),\n#                    \"C\": lambda x: x[x == 'C'].count()}\n# })\n# grouped_txns.columns = [\"_\".join(x) for x in grouped_txns.columns.ravel()]\n# grouped_txns = grouped_txns.reset_index()\n# # print (grouped_txns.shape)\n# print (grouped_txns.head(5))\n# # print (list(grouped_txns.columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bd400b364645c295b3dd9e971172d303a049d05"},"cell_type":"code","source":"# grouped_txns.to_pickle(\"grouped_txns.pickle\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"592a12779962decd83ae9e19ca659fcef97108e7","scrolled":false},"cell_type":"code","source":"train_data = train.join(grouped_txns.set_index(\"card_id\"), on=\"card_id\")\ntest_data = test.join(grouped_txns.set_index(\"card_id\"), on=\"card_id\")\n\n# print (train_data.shape)\n# print (test_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99fb8297efc7be47262bc382e4fdce11e777fe1e"},"cell_type":"code","source":"# Extracting number of active months\n\ntrain_data['first_active_month'] = pd.to_datetime(train_data['first_active_month'])\ntest_data['first_active_month'] = pd.to_datetime(test_data['first_active_month'])\ncurrent_date = datetime.datetime.today()\n\ndef _number_of_active_months(design_matrix):\n    design_matrix['number_of_active_months'] = ((current_date - design_matrix['first_active_month']) / 30.).dt.days\n    return design_matrix\n\ntrain_mod = _number_of_active_months(train_data)\ntest_mod = _number_of_active_months(test_data)\n\n## Imputing missing values in test data\nfill_na_with_mean = np.mean(test_mod['number_of_active_months'])\ntest_mod['number_of_active_months'].fillna(fill_na_with_mean, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7abaa4f5e808eead58de239970b1e8a7530012d0"},"cell_type":"code","source":"# Normalizing continuous features\n\nfeatures_to_norm = ['number_of_active_months', 'month_lag_sum', 'month_lag_mean',\n                    'installments_sum', 'installments_mean', 'authorized_flag_Y',\n                    'authorized_flag_N', 'category_1_Y', 'category_1_N',\n                    'category_3_A', 'category_3_B', 'category_3_C']\nfor feature in features_to_norm:\n    mean_value = np.mean(train_mod[feature])\n    stdev = np.std(train_mod[feature])\n    train_mod[feature] = train_mod[feature].apply(lambda x: (x - mean_value) / (stdev))\n    test_mod[feature] = test_mod[feature].apply(lambda x: (x - mean_value) / (stdev))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb61f262cafe6815d6a92b99d041b550b16ea6bf"},"cell_type":"code","source":"# Creating dummies for categorical features\n\ncat_features = ['feature_1', 'feature_2', 'feature_3']\nresponse = ['target']\n\ndef _create_dummies_for_categorical_features(design_matrix):\n    \"\"\"Create dummies for categorical features\"\"\"\n    design_matrix = pd.get_dummies(design_matrix, prefix=cat_features,\n                                   columns=cat_features)\n    return design_matrix\n\ndesign_matrix_train = train_mod[list(test_mod.columns) + response]\ndesign_matrix_test = test_mod[list(test_mod.columns)]\n\ntrain_with_dummies = _create_dummies_for_categorical_features(design_matrix_train)\ntest_with_dummies = _create_dummies_for_categorical_features(design_matrix_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dcc85be94fde43f4bfe5755886f1177a67c7e7a"},"cell_type":"code","source":"# Defining predictors for the model\n\n# predictors = ['purchase_amount_mean', 'category_1_Y', 'category_1_N',\n#               'month_lag_mean', 'installments_mean',\n#               'authorized_flag_N', 'authorized_flag_Y', 'number_of_active_months',\n#               'feature_2_1', 'feature_2_2', 'feature_2_3', 'feature_3_0', 'feature_3_1',\n#               'category_3_A', 'category_3_B', 'category_3_C',\n#               'feature_1_1', 'feature_1_2', 'feature_1_3', 'feature_1_4', 'feature_1_5']\n\npredictors = [x for x in test_with_dummies.columns if x not in [\n    'card_id', 'first_active_month', 'feature_2_1', 'feature_2_2']]\nprint (len(predictors))\nprint (predictors)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc9b66cf64b4b5e34b6dea29a9da928dc20c7033"},"cell_type":"code","source":"# xgb_param = xgb1.get_xgb_params()\n# xgtrain = xgb.DMatrix(train_with_dummies[predictors], label=train_with_dummies['target'])\n# cv_result = xgb.cv(xgb_param, xgtrain, num_boost_round=xgb1.get_params()['n_estimators'],\n#                    nfold=5, metrics='rmse', early_stopping_rounds=50, verbose_eval=True)\n# xgb1.set_params(n_estimators=cv_result.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e344782b15a732bf812d7487fea1d0989fa64e2c"},"cell_type":"code","source":"# print (cv_result[['train-rmse-mean', 'test-rmse-mean']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29aa12ec34b821887cc305d2fcdc662087dcff1f"},"cell_type":"code","source":"# xgb1.fit(train_with_dummies[predictors], train_with_dummies['target'], eval_metric='rmse')\n# dtrain_predictions = xgb1.predict(train_with_dummies[predictors])\n# print (\"MSE: %.4g\" % metrics.mean_squared_error(train_with_dummies['target'], dtrain_predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89c2b78f3f074a0bad07dcd50b49e33e3a4b3394","scrolled":false},"cell_type":"code","source":"# Model assessment using cross-validation\n\nregressor = XGBRegressor(seed=9)\nparameters = {'learning_rate': [0.01],\n              'n_estimators': [600],\n              'max_depth': [4],\n              'min_child_weight': [1],\n              'gamma': [0],\n              'subsample': [0.9],\n              'colsample_bytree': [0.8],\n              'objective': ['reg:linear'],\n              'reg_alpha': [0.01]\n             }\n\nmodel_grid = GridSearchCV(regressor,\n                          param_grid=parameters,\n                          scoring= \"neg_mean_squared_error\",\n                          cv=10,\n                          n_jobs=-1,\n                          verbose=2,\n                          iid=False\n                         )\n\nmodel_grid.fit(train_with_dummies[predictors], train_with_dummies['target'])\nprint (\"Best: %f using %s\" % (model_grid.best_score_, model_grid.best_params_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"c28686bec479a79b4cc7264d3f1b3c1c132e715b"},"cell_type":"code","source":"# Cross-validation results\n\ncv_results = model_grid.cv_results_\nresults = DataFrame.from_dict(cv_results, orient='columns')\nresults.sort_values(['mean_test_score'], ascending=False, inplace=True)\nprint (results[['param_n_estimators', 'mean_test_score']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a930db911ed1f711b7b109994085494888377656"},"cell_type":"code","source":"## Validation using best estimator\n\n# best_model = model_grid.best_estimator_\n# validation_data['target_predicted'] = best_model.predict(validation_data[predictors])\n# validation_data.to_csv('validation_data.csv', index=False)\n\n# RMSE\nmean_squared_error = (-1.) * model_grid.best_score_\nrmse = np.sqrt(mean_squared_error)\nprint ('RMSE: ', rmse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30e078ffa0130f3359505fe0518402b97d785d7d","scrolled":false},"cell_type":"code","source":"# Fitting model with the best (chosen) estimator\n\nrcParams['figure.figsize'] = 20, 8\nbest_model = model_grid.best_estimator_\nmodel = best_model.fit(train_with_dummies[predictors], train_with_dummies['target'])\nfeat_imp = pd.Series(model.feature_importances_, predictors).sort_values(ascending=False)\nfeat_imp.plot(kind='bar', title='Feature Importances')\nplt.ylabel('Feature Importance Score')\n# feat_imp = feat_imp[:99]\n# feat_imp.drop(\"feature_2_2\", inplace=True)\nprint (list(feat_imp.index))\nprint (feat_imp)\nprint (feat_imp.sum())\nfeat_imp.plot(kind='bar', title='Feature Importances')\nplt.ylabel('Feature Importance Score')\n\n# Model score\n# score = best_model.score(train_with_dummies[predictors], train_with_dummies['target'])\n# print ()\n# print ('Model score: ', score)\n\n# Predictions on test data\ntest_with_dummies['target'] = model.predict(test_with_dummies[predictors])\n# print (test_with_dummies.head(10))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Submissions\n\nsubmission_file = test_with_dummies[['card_id', 'target']]\n# print (submission_file.head(20))\n\nsubmission_file.to_csv('submission_26_' + str(rmse) + '.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}