{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport xgboost as xgb\nfrom sklearn import model_selection, preprocessing\nimport warnings\nimport time\nimport sys\nimport datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\npd.set_option('display.max_columns', 500)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"7a7877dff5c337c09ca111cdcbf527362c9217c7"},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6b5a505bfecabd70c6dac0ee30386d074b03953"},"cell_type":"code","source":"new_transactions = pd.read_csv('../input/new_merchant_transactions.csv',\n                               parse_dates=['purchase_date'])\n\nhistorical_transactions = pd.read_csv('../input/historical_transactions.csv',\n                                      parse_dates=['purchase_date'])\n\ndef binarize(df):\n    for col in ['authorized_flag', 'category_1']:\n        df[col] = df[col].map({'Y':1, 'N':0})\n    return df\n\nhistorical_transactions = binarize(historical_transactions)\nnew_transactions = binarize(new_transactions)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"156acd1e83dfba2c2896561b75a6a5a7782cab1d"},"cell_type":"markdown","source":"We then load the main files, formatting the dates and extracting the target:"},{"metadata":{"trusted":true,"_uuid":"a24cf99ad6b0785b5af2b101e06b400e26360d1e"},"cell_type":"code","source":"\nfrom datetime import date\n\ntoday = date.today()\n\ndef read_data(input_file):\n    df = pd.read_csv(input_file)\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['elapsed_time'] = (today - df['first_active_month'].dt.date).dt.days\n    return df\n#_________________________________________\ntrain = read_data('../input/train.csv')\ntest = read_data('../input/test.csv')\n\ntarget = train['target']\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3c2a5853f29c314dca418e1ad8420358fbf74fe"},"cell_type":"markdown","source":"<a id=\"2\"></a> <br>\n## Feature engineering\n"},{"metadata":{"trusted":true,"_uuid":"f787e3f81121b236528b59d0e0a723bcef510e4b"},"cell_type":"code","source":"historical_transactions['month_diff'] = ((datetime.datetime.today() - historical_transactions['purchase_date']).dt.days)//30\nhistorical_transactions['month_diff'] += historical_transactions['month_lag']\n\nnew_transactions['month_diff'] = ((datetime.datetime.today() - new_transactions['purchase_date']).dt.days)//30\nnew_transactions['month_diff'] += new_transactions['month_lag']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9db35828b8cb62fbe95a9d30e63b30b49966a304"},"cell_type":"code","source":"historical_transactions[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1da58e20da9fbb6d200119b242c168ae8fc5843"},"cell_type":"code","source":"\nhistorical_transactions = pd.get_dummies(historical_transactions, columns=['category_2', 'category_3'])\nnew_transactions = pd.get_dummies(new_transactions, columns=['category_2', 'category_3'])\n\nhistorical_transactions = reduce_mem_usage(historical_transactions)\nnew_transactions = reduce_mem_usage(new_transactions)\n\nagg_fun = {'authorized_flag': ['mean']}\nauth_mean = historical_transactions.groupby(['card_id']).agg(agg_fun)\nauth_mean.columns = ['_'.join(col).strip() for col in auth_mean.columns.values]\nauth_mean.reset_index(inplace=True)\n\nauthorized_transactions = historical_transactions[historical_transactions['authorized_flag'] == 1]\nhistorical_transactions = historical_transactions[historical_transactions['authorized_flag'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76c41ff3af21c7bfa9194f873f6f47c5849eb4d8"},"cell_type":"code","source":"historical_transactions['purchase_month'] = historical_transactions['purchase_date'].dt.month\nauthorized_transactions['purchase_month'] = authorized_transactions['purchase_date'].dt.month\nnew_transactions['purchase_month'] = new_transactions['purchase_date'].dt.month","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7493b32cb783d6fb6afdad60964eb41c9e42c2e3"},"cell_type":"markdown","source":"Then I define two functions that aggregate the info contained in these two tables. The first function aggregates the function by grouping on `card_id`:"},{"metadata":{"trusted":true,"_uuid":"82c25c7cd0d075195fb7bd63211c66f0dac9304b"},"cell_type":"code","source":"def aggregate_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n    'category_1': ['sum', 'mean'],\n    'category_2_1.0': ['mean'],\n    'category_2_2.0': ['mean'],\n    'category_2_3.0': ['mean'],\n    'category_2_4.0': ['mean'],\n    'category_2_5.0': ['mean'],\n    'category_3_A': ['mean'],\n    'category_3_B': ['mean'],\n    'category_3_C': ['mean'],\n    'merchant_id': ['nunique'],\n    'merchant_category_id': ['nunique'],\n    'state_id': ['nunique'],\n    'city_id': ['nunique'],\n    'subsector_id': ['nunique'],\n    'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n    'installments': ['sum', 'mean', 'max', 'min', 'std'],\n    'purchase_month': ['mean', 'max', 'min', 'std'],\n    'purchase_date': [np.ptp, 'min', 'max'],\n    'month_lag': ['mean', 'max', 'min', 'std'],\n    'month_diff': ['mean']\n    }\n    \n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['_'.join(col).strip() for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44ea0524d0af0001c5ea57a6593be35e2402d0b6"},"cell_type":"code","source":"history = aggregate_transactions(historical_transactions)\nhistory.columns = ['hist_' + c if c != 'card_id' else c for c in history.columns]\nhistory[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be09e1a7d4f307beb59c18bf61261239877da4e3"},"cell_type":"code","source":"authorized = aggregate_transactions(authorized_transactions)\nauthorized.columns = ['auth_' + c if c != 'card_id' else c for c in authorized.columns]\nauthorized[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78b87a7863b62fa3758ef3167504d3ad8c279f3d","scrolled":true},"cell_type":"code","source":"new = aggregate_transactions(new_transactions)\nnew.columns = ['new_' + c if c != 'card_id' else c for c in new.columns]\nnew[:5]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dcf0403c10b8ee817257a51e5edf8f1f81fcd593"},"cell_type":"markdown","source":"<a id=\"3\"></a> <br>\n## 3. Training the model\nWe now train the model with the features we previously defined. A first step consists in merging all the dataframes:"},{"metadata":{"trusted":true,"_uuid":"d27264c6c7f0af6af7ba141177bfd38f7a68dec3"},"cell_type":"code","source":"train = pd.merge(train, history, on='card_id', how='left')\ntest = pd.merge(test, history, on='card_id', how='left')\n\ntrain = pd.merge(train, authorized, on='card_id', how='left')\ntest = pd.merge(test, authorized, on='card_id', how='left')\n\ntrain = pd.merge(train, new, on='card_id', how='left')\ntest = pd.merge(test, new, on='card_id', how='left')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f20b37597afcb73107784b50a1ec8391bfda04a"},"cell_type":"code","source":"#card_id is encoded from string to numeric value\nfrom sklearn.preprocessing import LabelEncoder \nlbe = LabelEncoder()\nlbe = lbe.fit(train['card_id'])\ncard_id = lbe.transform(train['card_id']) \ntrain['card_id'] = card_id \ntrain['card_id'].head()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdddea93bc2f19e0eba3346559c41363ef17b502"},"cell_type":"code","source":"lbe_activation = LabelEncoder()\nlbe = lbe_activation.fit(train['first_active_month'])\nfirst_active_month = lbe.transform(train['first_active_month']) \ntrain['first_active_month'] = first_active_month \ntrain['first_active_month'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1b299aec7d02e6978005ffafabbeaf3c0477623"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ny = train['target']\ndel train['target']\ndel train ['first_active_month']\n\nX_train, X_test, y_train, y_test = train_test_split(train, y, \n                                                    test_size=0.3, \n                                                    random_state=1234)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99ab427e9676ffccf30833fcab543e6f77d15091"},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nmodel = xgb.XGBRegressor()\nparameters = {'nthread':[5], \n              'objective':['reg:linear'],\n              'learning_rate': [.03, 0.05, .07],\n              'max_depth': [5, 6, 7, 8],\n              'min_child_weight': [4],\n              'silent': [1],\n              'subsample': [0.7],\n              'colsample_bytree': [0.7],\n              'n_estimators': [100]}\n\nxgb_grid = GridSearchCV(model,\n                        parameters,\n                        cv = 5,\n                        n_jobs = 5,\n                        verbose=True) \n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"286000eabcfc1b48a6f33fe9d8c7a47d1eb1be40"},"cell_type":"code","source":"#this is used to get the best model and parameters\n#xgb_grid.fit(X_train, y_train)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d34df0200489aff931ccfdc9b7d28b003466aa10"},"cell_type":"code","source":"#print(xgb_grid.best_score_)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ffeff742553dc9fed64c7d1dac61e1f559b9007"},"cell_type":"code","source":"#print(xgb_grid.best_params_) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ce3ecc8ae748bbb629e92be346876833ec7a41b"},"cell_type":"code","source":"#train the model using the best score parameters \n'''\ncolsample_bytree: Subsample ratio of columns when constructing each tree\nlearning_rate: Boosting learning rate\nmax_depth : Maximum tree depth for base learners\nmin_child_weight: Minimum sum of instance weight(hessian) needed in a child.  \nn_estimators: Number of boosted trees to fit\nnthread: Number of parallel threads used to run xgboost\nobjective: Specify the learning task and the corresponding learning objective\nsilent: Whether to print messages while running boosting \nsubsample: Subsample ratio of the training instance\n'''\n\n'''model_final = xgb.XGBRegressor(colsample_bytree= 0.7, \n                               learning_rate= 0.03,\n                               max_depth= 7, \n                               min_child_weight= 4, \n                               n_estimators= 120,\n                               nthread= 4, \n                               objective= 'reg:linear',\n                               silent= 1, \n                               subsample= 0.7)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f309e0365e37337a7049482233d6d4a9839bf45"},"cell_type":"code","source":"model_final = xgb.XGBRegressor(colsample_bytree= 0.7, \n                               learning_rate= 0.05,\n                               max_depth= 6, \n                               min_child_weight= 4, \n                               n_estimators= 100,\n                               nthread= 5, \n                               objective= 'reg:linear',\n                               silent= 1, \n                               subsample= 0.7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a484518cc92076650e3deb435c77a84d77f69903"},"cell_type":"code","source":"model_final.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cf66d200ce1b045f93379ba6bf479d784ae2780"},"cell_type":"code","source":"# make predictions for test data\ny_pred = model_final.predict(X_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edbd6b74efcb55ab103825f8f4a748b17e411c08"},"cell_type":"code","source":"from math import sqrt\n\nrms = sqrt(mean_squared_error(y_test, y_pred))  \nprint(rms ) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d92bf90806de1f9634ab131c39d2f1956e7f8c5"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nxgb.plot_tree(model_final,num_trees=0)\nplt.rcParams['figure.figsize'] = [100, 50]\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62e51bf0f53e785540002189327540bfcc356e61"},"cell_type":"code","source":"xgb.plot_tree(model_final, num_trees=0, rankdir='LR')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"568ebac9b7fe8a2923d7bcf55ff49ab45b5ec659"},"cell_type":"code","source":"#feature importance \nxgb.plot_importance(model_final)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"003ae1b1bd522b1b0d992ff220ed98d2a6d7477a"},"cell_type":"markdown","source":"<a id=\"5\"></a> <br>\n## 5. Submission\n"},{"metadata":{"trusted":true,"_uuid":"6aeb9b87d848dd0e531596817a0da3c6e4abac3a"},"cell_type":"code","source":"#predict  score for test data\ndel test['first_active_month'] \n\nlbe_test = LabelEncoder()\nlbe_test = lbe_test.fit(test['card_id'])\ncard_id = lbe_test.transform(test['card_id']) \ntest['card_id'] = card_id \ntest['card_id'].head() \n\n\nfinal_pred = model_final.predict(test)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82d5ac08a13603b2a66c59d98584c4b709daee2d"},"cell_type":"code","source":"test2 = read_data('../input/test.csv')\n\nsub_df = pd.DataFrame({\"card_id\":test2[\"card_id\"].values})\nsub_df[\"target\"] = final_pred\nsub_df.to_csv(\"submit.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}