{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nplt.rcParams['figure.figsize'] = (8,6)\nplt.style.use('fivethirtyeight')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd501cbdc0fa4c54dd6fc1798d00f7db37e74d48","trusted":false},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\ntest = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f19efbda21451a50eeb9028cdf2006f9f7249b6d","trusted":false},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e042a0a88d3f04ff3caa451166de69dff86f1a9","trusted":false},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91f3d9715d56fd08abb1728035143b900e4c5e65","trusted":false},"cell_type":"code","source":"target_col = 'target'\n\nplt.scatter(range(train.shape[0]), np.sort(train[target_col].values))\nplt.xlabel('index', fontsize=12)\nplt.ylabel('Loyalty Score', fontsize=12)\nplt.title(\"Loyalty score on target\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4405939154707b48f0c4f16f9f497f3b9dc6c960","trusted":false},"cell_type":"code","source":"sns.distplot(train[target_col].values, bins=50, kde=False, color=\"red\")\nplt.title(\"Histogram of Loyalty score\")\nplt.xlabel('Loyalty score', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"295378780d7687b04a81eda040ebf950ced8d2c4"},"cell_type":"markdown","source":"Some of the loyalty values are far apart (less than -30) compared to others. Let us just get their count."},{"metadata":{"_uuid":"103f3d90f00ad5933ce0db8480f4318a5b141166","trusted":false},"cell_type":"code","source":"train[train['target']<-30]['target'].count()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"82662099f703dda767bd9c416ec7992726b405a8","trusted":false},"cell_type":"code","source":"cnt_srs = train['first_active_month'].dt.date.value_counts()\ncnt_srs = cnt_srs.sort_index()\nplt.figure(figsize=(14,6))\nsns.barplot(cnt_srs.index, cnt_srs.values, alpha=0.8, color='green')\nplt.xticks(rotation='vertical')\nplt.xlabel('First active month', fontsize=12)\nplt.ylabel('Number of cards', fontsize=12)\nplt.title(\"First active month count in train set\")\nplt.show()\n\n\ncnt_srs = test['first_active_month'].dt.date.value_counts()\ncnt_srs = cnt_srs.sort_index()\nplt.figure(figsize=(14,6))\nsns.barplot(cnt_srs.index, cnt_srs.values, alpha=0.8, color='green')\nplt.xticks(rotation='vertical')\nplt.xlabel('First active month', fontsize=12)\nplt.ylabel('Number of cards', fontsize=12)\nplt.title(\"First active month count in test set\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7bcdbd87f3ea8abfd0c79817b4303999d300efac"},"cell_type":"markdown","source":"We have similar distribution in both test and train dataset"},{"metadata":{"_uuid":"069a6ea4269bfa8a00d3d07f74855333b0b7a59b"},"cell_type":"markdown","source":"Distribution of the feature 1,2,3"},{"metadata":{"_kg_hide-input":true,"_uuid":"128acbd2ebc03294a9b5e672fdb75783733a1296","trusted":false},"cell_type":"code","source":"# feature 1\n\nsns.violinplot(x=\"feature_1\", y=target_col, data=train)\nplt.xticks(rotation='vertical')\nplt.xlabel('Feature 1', fontsize=12)\nplt.ylabel('Loyalty score', fontsize=12)\nplt.title(\"Feature 1 distribution\")\nplt.show()\n\n# feature 2\n\nsns.violinplot(x=\"feature_2\", y=target_col, data=train)\nplt.xticks(rotation='vertical')\nplt.xlabel('Feature 2', fontsize=12)\nplt.ylabel('Loyalty score', fontsize=12)\nplt.title(\"Feature 2 distribution\")\nplt.show()\n\n# feature 3\n\nsns.violinplot(x=\"feature_3\", y=target_col, data=train)\nplt.xticks(rotation='vertical')\nplt.xlabel('Feature 3', fontsize=12)\nplt.ylabel('Loyalty score', fontsize=12)\nplt.title(\"Feature 3 distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19f9d53c45d088ef7e1a12f7ee3b6e6d86df1ad4"},"cell_type":"markdown","source":"Similar kind of distribution on all 3"},{"metadata":{"_uuid":"04a4e6d985b97c6406639dbe328635ade872a98e","trusted":false},"cell_type":"code","source":"import datetime\ntrain['elapsed_time'] = (datetime.date(2018, 2, 1) - train['first_active_month'].dt.date).dt.days\ntest['elapsed_time'] = (datetime.date(2018, 2, 1) - test['first_active_month'].dt.date).dt.days","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3dd95721c762e60275d5376a169c847bf0691bd8","trusted":false},"cell_type":"code","source":"train['month'] = train.first_active_month.dt.month\ntrain['year'] = train.first_active_month.dt.year\ntest['month'] = test.first_active_month.dt.month\ntest['year'] = test.first_active_month.dt.year","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aef2608b2e567efff900d183f9f0868c27202645"},"cell_type":"markdown","source":"**Historical Transactions:**\nNow let us look at the historical transactions data for the cards."},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"7adbada14dd86acb51c8b41cbfd33c9866124616","trusted":false},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7906cda6d8083720d08cbefeb9c76aa5d460303","trusted":false},"cell_type":"code","source":"def binarize(df):\n    for col in ['authorized_flag', 'category_1']:\n        df[col] = df[col].map({'Y':1, 'N':0})\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02d5f3526d7385aeb29f1d838d34eb134287bfb9","trusted":false},"cell_type":"code","source":"holidays = [\n        ('Christmas_Day_2017', '2017-12-25'),  # Christmas: December 25 2017\n        ('Mothers_Day_2017', '2017-06-04'),  # Mothers Day: May 14 2017\n        ('fathers_day_2017', '2017-08-13'),  # fathers day: August 13 2017\n        ('Children_day_2017', '2017-10-12'),  # Childrens day: October 12 2017\n        ('Valentine_Day_2017', '2017-06-12'),  # Valentine's Day : 12th June, 2017\n        ('Black_Friday_2017', '2017-11-24'),  # Black Friday: 24th November 2017\n        ('Mothers_Day_2018', '2018-05-13'),\n    ]\n\ndef dist_holiday(df, col_name, date_holiday, date_ref, period=100):\n    df[col_name] = np.maximum(np.minimum((pd.to_datetime(date_holiday) - df[date_ref]).dt.days, period), 0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70db6c64d66585e19c72f1a0c4c90c63e5ed9479","trusted":false},"cell_type":"code","source":"historical = pd.read_csv(\"../input/historical_transactions.csv\", parse_dates=['purchase_date'])\nhistorical = binarize(historical)\nhistorical = pd.get_dummies(historical, columns=['category_2', 'category_3'])\nhistorical = reduce_mem_usage(historical)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"787e66f02f10a94b8edc3c209de509055a208b98","trusted":false},"cell_type":"code","source":"gdf = historical.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].size().reset_index()\ngdf.columns = [\"card_id\", \"historical_transactions\"]\ntrain = pd.merge(train, gdf, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"eddad5ae943c6f6d871d77ec425fd984a7136bb8","trusted":false},"cell_type":"code","source":"cnt_srs = train.groupby(\"historical_transactions\")['target'].mean()\ncnt_srs = cnt_srs.sort_index()\ncnt_srs = cnt_srs[:-50]\n\nsns.scatterplot(data=cnt_srs)\nplt.title('Loyalty score by Number of historical transactions')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"24b55658dbe73adc8cb75ea4207a636d16b4781a","trusted":false},"cell_type":"code","source":"for d_name, d_day in holidays:\n    dist_holiday(historical, d_name, d_day, 'purchase_date')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"cc6b1da26cd1486d1f2b2972215df5e86bb42a86","trusted":false},"cell_type":"code","source":"agg_func = {\n        'category_1': ['sum', 'mean'],\n        'category_2_1.0': ['sum', 'mean'],\n        'category_2_2.0': ['sum', 'mean'],\n        'category_2_3.0': ['sum', 'mean'],\n        'category_2_4.0': ['sum', 'mean'],\n        'category_2_5.0': ['sum', 'mean'],\n        'category_3_A': ['sum', 'mean'],\n        'category_3_B': ['sum', 'mean'],\n        'category_3_C': ['sum', 'mean'],\n        'authorized_flag': ['nunique', 'mean', 'sum'],\n        'merchant_id': ['nunique'],\n        'merchant_category_id': ['nunique'],\n        'state_id': ['nunique'],\n        'city_id': ['nunique'],\n        'subsector_id': ['nunique'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min', 'std','skew'],\n        'installments': ['sum', 'mean', 'max', 'min', 'std', 'skew'],\n        'purchase_date': [np.ptp, 'min', 'max'],\n        'month_lag': ['min', 'max'],\n        'Christmas_Day_2017': ['mean', 'sum'],\n        'Mothers_Day_2017': ['mean', 'sum'],\n        'fathers_day_2017': ['mean', 'sum'],\n        'Children_day_2017': ['mean', 'sum'],\n        'Valentine_Day_2017': ['mean', 'sum'],\n        'Black_Friday_2017': ['mean', 'sum'],\n        'Mothers_Day_2018': ['mean', 'sum']\n        }","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"230f72a9f48a92ec37a3b005cc210583f2a09064","trusted":false},"cell_type":"code","source":"# Adding more features from historical transactions\n\nhistorical.loc[:, 'purchase_date'] = pd.DatetimeIndex(historical['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\ngdf = historical.groupby(\"card_id\").agg(agg_func)\ngdf.columns = ['_historical_'.join(col).strip() for col in gdf.columns.values]\ngdf.reset_index(inplace=True)\n\ntrain = pd.merge(train, gdf, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6dc4436b5ffc377b383771924edd85ed40bfc3e"},"cell_type":"markdown","source":"**New Merchant Transactions**:\nIn this section, let us look at the new merchant transactions data and do some analysis"},{"metadata":{"_uuid":"ded9265245db82ac83b0d6dd191416d4e737264d","trusted":false},"cell_type":"code","source":"new= pd.read_csv(\"../input/new_merchant_transactions.csv\", parse_dates=['purchase_date'])\nnew = binarize(new)\nnew = pd.get_dummies(new, columns=['category_2', 'category_3'])\nnew = reduce_mem_usage(new)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"913af4cfc6120ba9073f73e0d335672ba26d7232","trusted":false},"cell_type":"code","source":"gdf = new.groupby(\"card_id\")\ngdf = gdf[\"purchase_amount\"].size().reset_index()\ngdf.columns = [\"card_id\", \"new_transactions\"]\ntrain = pd.merge(train, gdf, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"1a35aacff5697e1bc6c5b878eee2e15939f5e2b1","trusted":false},"cell_type":"code","source":"cnt_srs = train.groupby(\"new_transactions\")[target_col].mean()\ncnt_srs = cnt_srs.sort_index()\n\nsns.scatterplot(data=cnt_srs, size=(20,15))\nplt.title('Loyalty score by Number of new merchant transactions')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e9c806dff045accc8c6b01702df96ad5898d198","trusted":false},"cell_type":"code","source":"for d_name, d_day in holidays:\n    dist_holiday(new, d_name, d_day, 'purchase_date')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90d505e196f3d2a42f918e0fbae9d613a3295481","trusted":false},"cell_type":"code","source":"# Adding more features from new transactions\nnew.loc[:, 'purchase_date'] = pd.DatetimeIndex(new['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\ngdf = new.groupby(\"card_id\").agg(agg_func)\ngdf.columns = ['_new_'.join(col).strip() for col in gdf.columns.values]\ngdf.reset_index(inplace=True)\n\ntrain = pd.merge(train, gdf, on=\"card_id\", how=\"left\")\ntest = pd.merge(test, gdf, on=\"card_id\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce9c56a022d6078aa2e9c805244bef8b2ba040a3","trusted":false},"cell_type":"code","source":"del new, historical","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"4f498f9534be315992c1955c5ceb835001d76309","trusted":false},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7833e3822f026a7519b3db49891724ba70ac6970"},"cell_type":"markdown","source":"**Baseline Model**\n\nLet us build a baseline model using the features created so far."},{"metadata":{"_uuid":"dd56dc5cf44b3534577cbae83f8dd1a482640890","trusted":false},"cell_type":"code","source":"target = train['target']\ndel train['target']\n\nfeatures = [c for c in train.columns if c not in ['card_id', 'first_active_month']]\ncategorical_feats = [c for c in features if 'feature_' in c]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"b5b3824f4717f525323b1dce5fd9a98a2489a7a9","trusted":false},"cell_type":"code","source":"xgb_params = {\n            'gpu_id': 0,  \n            'objective': 'reg:linear', \n            'eval_metric': 'rmse', \n            'silent': True, \n            'booster': 'gbtree', \n            'n_jobs': 4, \n            'tree_method': 'gpu_hist', \n            'grow_policy': 'lossguide', \n            'max_depth': 12, \n            'seed': 538, \n            'colsample_bylevel': 0.9, \n            'colsample_bytree': 0.8, \n            'gamma': 0.0001, \n            'learning_rate': 0.006150886706231842, \n            'max_bin': 128, \n            'max_leaves': 47, \n            'min_child_weight': 40, \n            'reg_alpha': 10.0, \n            'reg_lambda': 10.0, \n            'subsample': 0.9,\n            'n_estimators': 20000\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6693d01f5641c78fb0844ff089753014ef91cb48","trusted":true},"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\nfolds = KFold(n_splits=10, shuffle=True, random_state=15)\noof = np.zeros(len(train))\nxgb_predictions = np.zeros(len(test))\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)):\n    print(\"fold n°{}\".format(fold_))\n    X_train, y_train = (train.iloc[trn_idx][features], target.iloc[trn_idx])\n    X_valid, y_valid = (train.iloc[val_idx][features], target.iloc[val_idx])\n    \n    clf = xgb.XGBRegressor(**xgb_params)\n    clf.fit(X_train, y_train, eval_set = [(X_valid, y_valid)], verbose=1000, early_stopping_rounds = 1000)\n    oof[val_idx] = clf.predict(X_valid, ntree_limit=clf.best_ntree_limit)\n    \n    xgb_predictions += clf.predict(test[features], ntree_limit=clf.best_ntree_limit) / folds.n_splits","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bfdd81cf1e835f1c47d530bef9acd70505f45986","trusted":false},"cell_type":"code","source":"print(\"CV score with XGB: {:<8.5f}\".format(mean_squared_error(oof, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc53ac4bfcf0573ff9faa067d507033f704f5e3d","trusted":false},"cell_type":"code","source":"xgb.plot_importance(clf, height=0.8, grid=False, title='XGBoost - Feature Importance', max_num_features=20)\nplt.figure(figsize=(20,18))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7bc930aee7fc7aecef59654ca879ce6594ab7b93","trusted":false},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test[\"card_id\"].values})\nsub_df[\"target\"] = xgb_predictions\nsub_df.to_csv(\"xgb_preds_updated.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,18))\nxgb.plot_tree(clf, num_trees=3)\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}