{"cells":[{"metadata":{"_uuid":"4305d3d9323daf4f8e9b3e65a2581dbd1e58abfb"},"cell_type":"markdown","source":"This kernel is inspired by Chau Ngoc Huynh's kernel (3.699).\n\nIntroducing the interaction on Categorical Variables followed by Stacking using Bayesian Ridge on Stratified K-Folds."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Import libraries\nimport numpy as np\nimport pandas as pd\nimport os\nimport time\nimport warnings\nimport gc\ngc.collect()\nimport os\nfrom six.moves import urllib\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nwarnings.filterwarnings('ignore')\n%matplotlib inline\nplt.style.use('seaborn')\nfrom scipy.stats import norm, skew\nfrom sklearn.preprocessing import StandardScaler\n\n# Scalers\nfrom sklearn.utils import shuffle\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.pipeline import FeatureUnion\n\n# Models\n\nfrom sklearn.linear_model import Lasso\nfrom sklearn.metrics import mean_squared_log_error,mean_squared_error, r2_score,mean_absolute_error\n\nfrom sklearn.model_selection import train_test_split \nfrom sklearn import metrics \nfrom sklearn.metrics import confusion_matrix \nfrom scipy.stats import reciprocal, uniform\n\nfrom sklearn.model_selection import StratifiedKFold, RepeatedKFold\n\n# Cross-validation\nfrom sklearn.model_selection import KFold, cross_val_predict, cross_val_score, cross_validate \n\n# GridSearchCV\nfrom sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n\n#Common data processors\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nfrom sklearn import feature_selection, model_selection, metrics\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.utils import check_array\nfrom scipy import sparse\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"np.random.seed(1234)\ngc.collect()\n\nplt.rcParams['axes.labelsize'] = 14\nplt.rcParams['xtick.labelsize'] = 12\nplt.rcParams['ytick.labelsize'] = 12","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6609981e38b1a1d8fef2357d10480fbd1447b9d"},"cell_type":"code","source":"#Reduce the memory usage - Inspired by Panchajanya Banerjee\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"666727f1e45ae33a030c8a784f6be415531cc355"},"cell_type":"code","source":"def one_hot_encoder(df, nan_as_category = True):\n    original_columns = list(df.columns)\n    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n    new_columns = [c for c in df.columns if c not in original_columns]\n    return df, new_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8196fd34b94e2c65341af1d2e66ea1ade3408b5"},"cell_type":"code","source":"train = reduce_mem_usage(pd.read_csv('../input/train.csv',parse_dates=[\"first_active_month\"]))\ntest = reduce_mem_usage(pd.read_csv('../input/test.csv', parse_dates=[\"first_active_month\"]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5001865ed5ae2b5f6854e81eff71694120921b1a"},"cell_type":"code","source":"train.shape, train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c46ddb12a4bb396ba9850a260ed015878feefb8f"},"cell_type":"code","source":"plt.subplot(1, 2, 2)\nsns.distplot(train.target, kde=True, fit = norm)\nplt.xlabel('Customer Loyality (Skewed)')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33266387e8c90a1ba3d5b6dd9d7ba3679c7947c9"},"cell_type":"code","source":"# Remove the Outliers if any \ntrain['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\ntrain['outliers'].value_counts()\n\nfor features in ['feature_1','feature_2','feature_3']:\n    order_label = train.groupby([features])['outliers'].mean()\n    train[features] = train[features].map(order_label)\n    test[features] =  test[features].map(order_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"714ddd43973658ebf7f90593b7526ebddb9debda"},"cell_type":"code","source":"# Now extract the month, year, day, weekdayss\ntrain[\"month\"] = train[\"first_active_month\"].dt.month\ntrain[\"year\"] = train[\"first_active_month\"].dt.year\ntrain['week'] = train[\"first_active_month\"].dt.weekofyear\ntrain['dayofweek'] = train['first_active_month'].dt.dayofweek\ntrain['days'] = (datetime.date(2018, 2, 1) - train['first_active_month'].dt.date).dt.days\ntrain['quarter'] = train['first_active_month'].dt.quarter\ntrain['is_month_start'] = train['first_active_month'].dt.is_month_start\n\n#Interaction Variables\ntrain['days_feature1'] = train['days'] * train['feature_1']\ntrain['days_feature2'] = train['days'] * train['feature_2']\ntrain['days_feature3'] = train['days'] * train['feature_3']\n\ntrain['days_feature1_ratio'] = train['feature_1'] / train['days']\ntrain['days_feature2_ratio'] = train['feature_2'] / train['days']\ntrain['days_feature3_ratio'] = train['feature_3'] / train['days']\n\ntrain['feature_sum'] = train['feature_1'] + train['feature_2'] + train['feature_3']\ntrain['feature_mean'] = train['feature_sum']/3\ntrain['feature_max'] = train[['feature_1', 'feature_2', 'feature_3']].max(axis=1)\ntrain['feature_min'] = train[['feature_1', 'feature_2', 'feature_3']].min(axis=1)\ntrain['feature_var'] = train[['feature_1', 'feature_2', 'feature_3']].std(axis=1)\n\ntest[\"month\"] = test[\"first_active_month\"].dt.month\ntest[\"year\"] = test[\"first_active_month\"].dt.year\ntest['week'] = test[\"first_active_month\"].dt.weekofyear\ntest['dayofweek'] = test['first_active_month'].dt.dayofweek\ntest['days'] = (datetime.date(2018, 2, 1) - test['first_active_month'].dt.date).dt.days\ntest['quarter'] = test['first_active_month'].dt.quarter\ntest['is_month_start'] = test['first_active_month'].dt.is_month_start\n\n#Interaction Variables\ntest['days_feature1'] = test['days'] * train['feature_1']\ntest['days_feature2'] = test['days'] * train['feature_2']\ntest['days_feature3'] = test['days'] * train['feature_3']\n\ntest['days_feature1_ratio'] = test['feature_1'] / train['days']\ntest['days_feature2_ratio'] = test['feature_2'] / train['days']\ntest['days_feature3_ratio'] = test['feature_3'] / train['days']\n\ntest['feature_sum'] = test['feature_1'] + test['feature_2'] + test['feature_3']\ntest['feature_mean'] = test['feature_sum']/3\ntest['feature_max'] = test[['feature_1', 'feature_2', 'feature_3']].max(axis=1)\ntest['feature_min'] = test[['feature_1', 'feature_2', 'feature_3']].min(axis=1)\ntest['feature_var'] = test[['feature_1', 'feature_2', 'feature_3']].std(axis=1)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6c5e2f5dfb476dd4ebe5faf920b41a6128e1023a"},"cell_type":"markdown","source":"## Feature Engineering\nFeatures from Transaction data"},{"metadata":{"trusted":true,"_uuid":"02863f839ba600ff838257ea74be06211604365f"},"cell_type":"code","source":"def aggregate_transaction_hist(trans, prefix):  \n        \n    agg_func = {\n        'purchase_amount' : ['sum','max','min','mean','var','skew'],\n        'installments' : ['sum','max','mean','var','skew'],\n        'purchase_date' : ['max','min'],\n        'month_lag' : ['max','min','mean','var','skew'],\n        'month_diff' : ['max','min','mean','var','skew'],\n        'weekend' : ['sum', 'mean'],\n        'authorized_flag': ['sum', 'mean'],\n        'category_1': ['sum','mean', 'max','min'],\n        'card_id' : ['size','count'],\n        'month': ['nunique', 'mean', 'min', 'max'],\n        'hour': ['nunique', 'mean', 'min', 'max'],\n        'weekofyear': ['nunique', 'mean', 'min', 'max'],\n        'dayofweek': ['nunique', 'mean', 'min', 'max'],\n        'year': ['nunique', 'mean', 'min', 'max'],\n        'subsector_id': ['nunique'],\n        'merchant_category_id' : ['nunique'],\n        'price' :['sum','mean','max','min','var'],\n        'duration' : ['mean','min','max','var','skew'],\n        'amount_month_ratio':['mean','min','max','var','skew']\n        \n    }\n    \n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcc0977281d1ba857e6132a0486d89c008408d42"},"cell_type":"code","source":"transactions = reduce_mem_usage(pd.read_csv('../input/historical_transactions.csv'))\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6d7fe3b156091de3ba285dad36455767329221d"},"cell_type":"code","source":"#impute missing values (excluded)\ntransactions['category_2'] = transactions['category_2'].fillna(1.0,inplace=True)\ntransactions['category_3'] = transactions['category_3'].fillna('A',inplace=True)\ntransactions['merchant_id'] = transactions['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)\ntransactions['installments'].replace(-1, np.nan,inplace=True)\ntransactions['installments'].replace(999, np.nan,inplace=True)\ntransactions['purchase_amount'] = transactions['purchase_amount'].apply(lambda x: min(x, 0.8))\n\n#Feature Engineering - Adding new features inspired by Chau's first kernel\ntransactions['authorized_flag'] = transactions['authorized_flag'].map({'Y': 1, 'N': 0})\ntransactions['category_1'] = transactions['category_1'].map({'Y': 1, 'N': 0})\ntransactions['category_3'] = transactions['category_3'].map({'A':0, 'B':1, 'C':2})\n\ntransactions['purchase_date'] = pd.to_datetime(transactions['purchase_date'])\ntransactions['year'] = transactions['purchase_date'].dt.year\ntransactions['weekofyear'] = transactions['purchase_date'].dt.weekofyear\ntransactions['month'] = transactions['purchase_date'].dt.month\ntransactions['dayofweek'] = transactions['purchase_date'].dt.dayofweek\ntransactions['weekend'] = (transactions.purchase_date.dt.weekday >=5).astype(int)\ntransactions['hour'] = transactions['purchase_date'].dt.hour \ntransactions['quarter'] = transactions['purchase_date'].dt.quarter\ntransactions['is_month_start'] = transactions['purchase_date'].dt.is_month_start\ntransactions['month_diff'] = ((datetime.datetime.today() - transactions['purchase_date']).dt.days)//30\ntransactions['month_diff'] += transactions['month_lag']\n\n# additional features\ntransactions['duration'] = transactions['purchase_amount']*transactions['month_diff']\ntransactions['amount_month_ratio'] = transactions['purchase_amount']/transactions['month_diff']\ntransactions['price'] = transactions['purchase_amount'] / transactions['installments']\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c078f67889b70fbcb66413cf8f4afd42d08425f5"},"cell_type":"code","source":"agg_func = {\n        'mean': ['mean'],\n    }\nfor col in ['category_2','category_3']:\n    transactions[col+'_mean'] = transactions['purchase_amount'].groupby(transactions[col]).agg('mean')\n    transactions[col+'_max'] = transactions['purchase_amount'].groupby(transactions[col]).agg('max')\n    transactions[col+'_min'] = transactions['purchase_amount'].groupby(transactions[col]).agg('min')\n    transactions[col+'_sum'] = transactions['purchase_amount'].groupby(transactions[col]).agg('sum')\n    agg_func[col+'_mean'] = ['mean']\n    \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f34d81d973133511957126db89e58e83a14d308"},"cell_type":"code","source":"merge_trans = aggregate_transaction_hist(transactions, prefix='hist_')\ndel transactions\ngc.collect()\ntrain = pd.merge(train, merge_trans, on='card_id',how='left')\ntest = pd.merge(test, merge_trans, on='card_id',how='left')\ndel merge_trans\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e31dc0f8e20ca2b3631848b52210c3313b91cb40"},"cell_type":"code","source":"#Adding new features inspired by Chau's first kernel\ntrain['hist_purchase_date_max'] = pd.to_datetime(train['hist_purchase_date_max'])\ntrain['hist_purchase_date_min'] = pd.to_datetime(train['hist_purchase_date_min'])\ntrain['hist_purchase_date_diff'] = (train['hist_purchase_date_max'] - train['hist_purchase_date_min']).dt.days\ntrain['hist_purchase_date_average'] = train['hist_purchase_date_diff']/train['hist_card_id_size']\ntrain['hist_purchase_date_uptonow'] = (datetime.datetime.today() - train['hist_purchase_date_max']).dt.days\ntrain['hist_purchase_date_uptomin'] = (datetime.datetime.today() - train['hist_purchase_date_min']).dt.days\ntrain['hist_first_buy'] = (train['hist_purchase_date_min'] - train['first_active_month']).dt.days\ntrain['hist_last_buy'] = (train['hist_purchase_date_max'] - train['first_active_month']).dt.days\n\nfor feature in ['hist_purchase_date_max','hist_purchase_date_min']:\n    train[feature] = train[feature].astype(np.int64) * 1e-9\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53fd1c4c82cf5cb1a20e4aadaf36d3911b6c553e"},"cell_type":"code","source":"#Adding new features inspired by Chau's first kernel\ntest['hist_purchase_date_max'] = pd.to_datetime(test['hist_purchase_date_max'])\ntest['hist_purchase_date_min'] = pd.to_datetime(test['hist_purchase_date_min'])\ntest['hist_purchase_date_diff'] = (test['hist_purchase_date_max'] - test['hist_purchase_date_min']).dt.days\ntest['hist_purchase_date_average'] = test['hist_purchase_date_diff']/test['hist_card_id_size']\ntest['hist_purchase_date_uptonow'] = (datetime.datetime.today() - test['hist_purchase_date_max']).dt.days\ntest['hist_purchase_date_uptomin'] = (datetime.datetime.today() - test['hist_purchase_date_min']).dt.days\n\ntest['hist_first_buy'] = (test['hist_purchase_date_min'] - test['first_active_month']).dt.days\ntest['hist_last_buy'] = (test['hist_purchase_date_max'] - test['first_active_month']).dt.days\n\nfor feature in ['hist_purchase_date_max','hist_purchase_date_min']:\n    test[feature] = test[feature].astype(np.int64) * 1e-9\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b150375233e8971a01fcc02cac9d5957c8a17cb3"},"cell_type":"code","source":"train.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5178dad335d4003280dced08af15ddbdcd946bc2"},"cell_type":"code","source":"def aggregate_transaction_new(trans, prefix):  \n        \n    agg_func = {\n        'purchase_amount' : ['sum','max','min','mean','var','skew'],\n        'installments' : ['sum','max','mean','var','skew'],\n        'purchase_date' : ['max','min'],\n        'month_lag' : ['max','min','mean','var','skew'],\n        'month_diff' : ['max','min','mean','var','skew'],\n        'weekend' : ['sum', 'mean'],\n        'authorized_flag': ['sum', 'mean'],\n        'category_1': ['sum','mean', 'max','min'],\n        'card_id' : ['size','count'],\n        'month': ['nunique', 'mean', 'min', 'max'],\n        'hour': ['nunique', 'mean', 'min', 'max'],\n        'weekofyear': ['nunique', 'mean', 'min', 'max'],\n        'dayofweek': ['nunique', 'mean', 'min', 'max'],\n        'year': ['nunique', 'mean', 'min', 'max'],\n        'subsector_id': ['nunique'],\n        'merchant_category_id' : ['nunique'],\n        'price' :['sum','mean','max','min','var'],\n        'duration' : ['mean','min','max','var','skew'],\n        'amount_month_ratio':['mean','min','max','var','skew']\n    }\n    \n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b90915c5e54f86d433fcec14743f4cabd4832de6"},"cell_type":"code","source":"new_transactions = reduce_mem_usage(pd.read_csv('../input/new_merchant_transactions.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6be0dba28b15f4ee40d4994d6f03c78c10ebec1"},"cell_type":"code","source":"#impute missing values\nnew_transactions['category_2'] = new_transactions['category_2'].fillna(1.0,inplace=True)\nnew_transactions['category_3'] = new_transactions['category_3'].fillna('A',inplace=True)\nnew_transactions['merchant_id'] = new_transactions['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)\nnew_transactions['installments'].replace(-1, np.nan,inplace=True)\nnew_transactions['installments'].replace(999, np.nan,inplace=True)\nnew_transactions['purchase_amount'] = new_transactions['purchase_amount'].apply(lambda x: min(x, 0.8))\n\n#Adding new features inspired by Chau's first kernel\nnew_transactions['authorized_flag'] = new_transactions['authorized_flag'].map({'Y': 1, 'N': 0})\nnew_transactions['category_1'] = new_transactions['category_1'].map({'Y': 1, 'N': 0})\nnew_transactions['category_3'] = new_transactions['category_3'].map({'A':0, 'B':1, 'C':2}) \n\nnew_transactions['purchase_date'] = pd.to_datetime(new_transactions['purchase_date'])\nnew_transactions['year'] = new_transactions['purchase_date'].dt.year\nnew_transactions['weekofyear'] = new_transactions['purchase_date'].dt.weekofyear\nnew_transactions['month'] = new_transactions['purchase_date'].dt.month\nnew_transactions['dayofweek'] = new_transactions['purchase_date'].dt.dayofweek\nnew_transactions['weekend'] = (new_transactions.purchase_date.dt.weekday >=5).astype(int)\nnew_transactions['hour'] = new_transactions['purchase_date'].dt.hour \nnew_transactions['quarter'] = new_transactions['purchase_date'].dt.quarter\nnew_transactions['is_month_start'] = new_transactions['purchase_date'].dt.is_month_start\nnew_transactions['month_diff'] = ((datetime.datetime.today() - new_transactions['purchase_date']).dt.days)//30\nnew_transactions['month_diff'] += new_transactions['month_lag']\n\ngc.collect()\n\n# additional features\nnew_transactions['duration'] = new_transactions['purchase_amount']*new_transactions['month_diff']\nnew_transactions['amount_month_ratio'] = new_transactions['purchase_amount']/new_transactions['month_diff']\nnew_transactions['price'] = new_transactions['purchase_amount'] / new_transactions['installments']\n\naggs = {\n        'mean': ['mean'],\n    }\n\nfor col in ['category_2','category_3']:\n    new_transactions[col+'_mean'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg('mean')\n    new_transactions[col+'_max'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg('max')\n    new_transactions[col+'_min'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg('min')\n    new_transactions[col+'_var'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg('var')\n    aggs[col+'_mean'] = ['mean']\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bf3521b3c01e31eee257f86717f5d7fd3d6c3ec"},"cell_type":"code","source":"merge_new = aggregate_transaction_new(new_transactions, prefix='new_')\ndel new_transactions\ngc.collect()\n\ntrain = pd.merge(train, merge_new, on='card_id',how='left')\ntest = pd.merge(test, merge_new, on='card_id',how='left')\ndel merge_new\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3e521c1040c34fefedbc4b1bc945712eef8b113"},"cell_type":"code","source":"#Adding new features inspired by Chau's first kernel\ntrain['new_purchase_date_max'] = pd.to_datetime(train['new_purchase_date_max'])\ntrain['new_purchase_date_min'] = pd.to_datetime(train['new_purchase_date_min'])\ntrain['new_purchase_date_diff'] = (train['new_purchase_date_max'] - train['new_purchase_date_min']).dt.days\ntrain['new_purchase_date_average'] = train['new_purchase_date_diff']/train['new_card_id_size']\ntrain['new_purchase_date_uptonow'] = (datetime.datetime.today() - train['new_purchase_date_max']).dt.days\ntrain['new_purchase_date_uptomin'] = (datetime.datetime.today() - train['new_purchase_date_min']).dt.days\ntrain['new_first_buy'] = (train['new_purchase_date_min'] - train['first_active_month']).dt.days\ntrain['new_last_buy'] = (train['new_purchase_date_max'] - train['first_active_month']).dt.days\nfor feature in ['new_purchase_date_max','new_purchase_date_min']:\n    train[feature] = train[feature].astype(np.int64) * 1e-9\n\n#Adding new features inspired by Chau's first kernel\ntest['new_purchase_date_max'] = pd.to_datetime(test['new_purchase_date_max'])\ntest['new_purchase_date_min'] = pd.to_datetime(test['new_purchase_date_min'])\ntest['new_purchase_date_diff'] = (test['new_purchase_date_max'] - test['new_purchase_date_min']).dt.days\ntest['new_purchase_date_average'] = test['new_purchase_date_diff']/test['new_card_id_size']\ntest['new_purchase_date_uptonow'] = (datetime.datetime.today() - test['new_purchase_date_max']).dt.days\ntest['new_purchase_date_uptomin'] = (datetime.datetime.today() - test['new_purchase_date_min']).dt.days\ntest['new_first_buy'] = (test['new_purchase_date_min'] - test['first_active_month']).dt.days\ntest['new_last_buy'] = (test['new_purchase_date_max'] - test['first_active_month']).dt.days\n\nfor feature in ['new_purchase_date_max','new_purchase_date_min']:\n    test[feature] = test[feature].astype(np.int64) * 1e-9\n    \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"931d044acc32f0081c5044dc6208a2a9280c3fa4"},"cell_type":"code","source":"#new features referred from https://www.kaggle.com/mfjwr1/simple-lightgbm-without-blending\ntrain['card_id_total'] = train['new_card_id_size']+train['hist_card_id_size']\ntrain['card_id_cnt_total'] = train['new_card_id_count']+train['hist_card_id_count']\ntrain['card_id_cnt_ratio'] = train['new_card_id_count']/train['hist_card_id_count']\ntrain['purchase_amount_total'] = train['new_purchase_amount_sum']+train['hist_purchase_amount_sum']\ntrain['purchase_amount_mean'] = train['new_purchase_amount_mean']+train['hist_purchase_amount_mean']\ntrain['purchase_amount_max'] = train['new_purchase_amount_max']+train['hist_purchase_amount_max']\ntrain['purchase_amount_min'] = train['new_purchase_amount_min']+train['hist_purchase_amount_min']\ntrain['purchase_amount_ratio'] = train['new_purchase_amount_sum']/train['hist_purchase_amount_sum']\ntrain['month_diff_mean'] = train['new_month_diff_mean']+train['hist_month_diff_mean']\ntrain['month_diff_ratio'] = train['new_month_diff_mean']/train['hist_month_diff_mean']\ntrain['month_lag_mean'] = train['new_month_lag_mean']+train['hist_month_lag_mean']\ntrain['month_lag_max'] = train['new_month_lag_max']+train['hist_month_lag_max']\ntrain['month_lag_min'] = train['new_month_lag_min']+train['hist_month_lag_min']\ntrain['category_1_mean'] = train['new_category_1_mean']+train['hist_category_1_mean']\ntrain['installments_total'] = train['new_installments_sum']+train['hist_installments_sum']\ntrain['installments_mean'] = train['new_installments_mean']+train['hist_installments_mean']\ntrain['installments_max'] = train['new_installments_max']+train['hist_installments_max']\ntrain['installments_ratio'] = train['new_installments_sum']/train['hist_installments_sum']\ntrain['price_total'] = train['purchase_amount_total'] / train['installments_total']\ntrain['price_mean'] = train['purchase_amount_mean'] / train['installments_mean']\ntrain['price_max'] = train['purchase_amount_max'] / train['installments_max']\ntrain['duration_mean'] = train['new_duration_mean']+train['hist_duration_mean']\ntrain['duration_min'] = train['new_duration_min']+train['hist_duration_min']\ntrain['duration_max'] = train['new_duration_max']+train['hist_duration_max']\ntrain['amount_month_ratio_mean']=train['new_amount_month_ratio_mean']+train['hist_amount_month_ratio_mean']\ntrain['amount_month_ratio_min']=train['new_amount_month_ratio_min']+train['hist_amount_month_ratio_min']\ntrain['amount_month_ratio_max']=train['new_amount_month_ratio_max']+train['hist_amount_month_ratio_max']\ntrain['new_CLV'] = train['new_card_id_count'] * train['new_purchase_amount_sum'] / train['new_month_diff_mean']\ntrain['hist_CLV'] = train['hist_card_id_count'] * train['hist_purchase_amount_sum'] / train['hist_month_diff_mean']\ntrain['CLV_ratio'] = train['new_CLV'] / train['hist_CLV']\n\ntest['card_id_total'] = test['new_card_id_size']+test['hist_card_id_size']\ntest['card_id_cnt_total'] = test['new_card_id_count']+test['hist_card_id_count']\ntest['card_id_cnt_ratio'] = test['new_card_id_count']/test['hist_card_id_count']\ntest['purchase_amount_total'] = test['new_purchase_amount_sum']+test['hist_purchase_amount_sum']\ntest['purchase_amount_mean'] = test['new_purchase_amount_mean']+test['hist_purchase_amount_mean']\ntest['purchase_amount_max'] = test['new_purchase_amount_max']+test['hist_purchase_amount_max']\ntest['purchase_amount_min'] = test['new_purchase_amount_min']+test['hist_purchase_amount_min']\ntest['purchase_amount_ratio'] = test['new_purchase_amount_sum']/test['hist_purchase_amount_sum']\ntest['month_diff_mean'] = test['new_month_diff_mean']+test['hist_month_diff_mean']\ntest['month_diff_ratio'] = test['new_month_diff_mean']/test['hist_month_diff_mean']\ntest['month_lag_mean'] = test['new_month_lag_mean']+test['hist_month_lag_mean']\ntest['month_lag_max'] = test['new_month_lag_max']+test['hist_month_lag_max']\ntest['month_lag_min'] = test['new_month_lag_min']+test['hist_month_lag_min']\ntest['category_1_mean'] = test['new_category_1_mean']+test['hist_category_1_mean']\ntest['installments_total'] = test['new_installments_sum']+test['hist_installments_sum']\ntest['installments_mean'] = test['new_installments_mean']+test['hist_installments_mean']\ntest['installments_max'] = test['new_installments_max']+test['hist_installments_max']\ntest['installments_ratio'] = test['new_installments_sum']/test['hist_installments_sum']\ntest['price_total'] = test['purchase_amount_total'] / test['installments_total']\ntest['price_mean'] = test['purchase_amount_mean'] / test['installments_mean']\ntest['price_max'] = test['purchase_amount_max'] / test['installments_max']\ntest['duration_mean'] = test['new_duration_mean']+test['hist_duration_mean']\ntest['duration_min'] = test['new_duration_min']+test['hist_duration_min']\ntest['duration_max'] = test['new_duration_max']+test['hist_duration_max']\ntest['amount_month_ratio_mean']=test['new_amount_month_ratio_mean']+test['hist_amount_month_ratio_mean']\ntest['amount_month_ratio_min']=test['new_amount_month_ratio_min']+test['hist_amount_month_ratio_min']\ntest['amount_month_ratio_max']=test['new_amount_month_ratio_max']+test['hist_amount_month_ratio_max']\ntest['new_CLV'] = test['new_card_id_count'] * test['new_purchase_amount_sum'] / test['new_month_diff_mean']\ntest['hist_CLV'] = test['hist_card_id_count'] * test['hist_purchase_amount_sum'] / test['hist_month_diff_mean']\ntest['CLV_ratio'] = test['new_CLV'] / test['hist_CLV']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba2065e58fc885df222e76da6c95ae819fba122f"},"cell_type":"code","source":"train.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3024d6838249c46f618065b8bf8b2a67df31b69f"},"cell_type":"code","source":"train = train.drop(['card_id', 'first_active_month'], axis = 1)\ntest = test.drop(['card_id', 'first_active_month'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17cde6e9c9ce2f635a99d2731a45619d5f6a1ec3"},"cell_type":"code","source":"# X and Y\ndf_train_columns = [c for c in train.columns if c not in ['first_active_month', 'target', 'card_id', 'outliers',\n                  'hist_purchase_date_max', 'hist_purchase_date_min', 'hist_card_id_size',\n                  'new_purchase_date_max', 'new_purchase_date_min', 'new_card_id_size']] \ntarget = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"753e52c1bf4c484f814e7a5c2fda636db75da7df"},"cell_type":"markdown","source":"# LGBM Model\nStratified K Folds enumerated on training set and outliers"},{"metadata":{"trusted":true,"_uuid":"be2031a01ea0a91d808091610825f960762c169e"},"cell_type":"code","source":"# Change in Hyper Parameters using kernel : https://www.kaggle.com/mfjwr1/simple-lightgbm-without-blending/output\nimport lightgbm as lgb\n\nfolds = StratifiedKFold(n_splits=5, shuffle = True, random_state=15)\noof = np.zeros(len(train))\npredictions = np.zeros(len(test))\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, train['outliers'].values)):\n    \n    print(\"fold {}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx][df_train_columns], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx][df_train_columns], label=target.iloc[val_idx])\n\n    param ={\n                'task': 'train',\n                'boosting': 'goss',\n                'objective': 'regression',\n                'metric': 'rmse',\n                'learning_rate': 0.01,\n                'subsample': 0.9855232997390695,\n                'max_depth': -1,\n                'top_rate': 0.9064148448434349,\n                'num_leaves': 31,\n                'min_child_weight': 41.9612869171337,\n                'other_rate': 0.0721768246018207,\n                'reg_alpha': 9.677537745007898,\n                'colsample_bytree': 0.5665320670155495,\n                'min_split_gain': 9.820197773625843,\n                'reg_lambda': 0.1,\n                'min_data_in_leaf': 32,\n                'verbose': -1,\n                'seed':int(2**fold_),\n                'bagging_seed':2015,\n                'drop_seed':int(2**fold_)\n                }\n    \n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)\n    oof[val_idx] = clf.predict(train.iloc[val_idx][df_train_columns], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = df_train_columns\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(test[df_train_columns], num_iteration=clf.best_iteration) / folds.n_splits\n\nnp.sqrt(mean_squared_error(oof, target))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02045ab2312edad8c2fb96f69948cfdd2e0feb9f"},"cell_type":"code","source":"print(\"CV score: {:<8.5f}\".format(mean_squared_error(oof, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3469193c1730c8f6bff3427f7df40ba685d54842"},"cell_type":"code","source":"# Feature importance\ncols = (feature_importance_df[[\"Feature\", \"importance\"]]\n        .groupby(\"Feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.Feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"Feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88bef9d39777f8d7865429f9df803b6291135517"},"cell_type":"code","source":"features = [c for c in train.columns if c not in ['first_active_month', 'target', 'card_id', 'outliers',\n                  'hist_purchase_date_max', 'hist_purchase_date_min', 'hist_card_id_size',\n                  'new_purchase_date_max', 'new_purchase_date_min', 'new_card_id_size']]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b9dd102d2440cd11da30f74eeec1d5ce597d65b"},"cell_type":"markdown","source":"LGBM - Repeated K Folds enumerated on training set and Target"},{"metadata":{"trusted":true,"_uuid":"06e43690b321b39ea676565954f7517ca6202dba"},"cell_type":"code","source":"folds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=2019)\noof_2 = np.zeros(len(train))\npredictions_2 = np.zeros(len(test))\nfeature_importance_df_2 = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)):\n    \n    print(\"fold {}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])\n    \n    param ={\n                'task': 'train',\n                'boosting': 'goss',\n                'objective': 'regression',\n                'metric': 'rmse',\n                'learning_rate': 0.75836372582243783,\n                'subsample': 0.91158142068248083,\n                'max_depth': 29,\n                'top_rate': 0.7790255922489042,\n                'num_leaves': 562,\n                'min_child_weight': 27,\n                'other_rate': 0.03564199289369395,\n                'reg_alpha': 0.72876375065913579,\n                'colsample_bytree': 0.83435723889734326,\n                'min_split_gain': 27.378180277455101,\n                'reg_lambda': 94.549009291544877,\n                'min_data_in_leaf': 21,\n                'verbose': -1,\n                'seed':20190208,\n                'bagging_seed':20190208,\n                'drop_seed':20190208\n                }\n        \n    num_round = 10000\n    clf_r = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)\n    oof_2[val_idx] = clf_r.predict(train.iloc[val_idx][features], num_iteration=clf_r.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = features\n    fold_importance_df[\"importance\"] = clf_r.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df_2 = pd.concat([feature_importance_df_2, fold_importance_df], axis=0)\n    \n    predictions_2 += clf_r.predict(test[features], num_iteration=clf_r.best_iteration) / (5 * 2)\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof_2, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c1ce586e48f735dab72a81f34f233c57c9565f5"},"cell_type":"markdown","source":"Feature importance - Repeated K Folds"},{"metadata":{"trusted":true,"_uuid":"c1a8d85602872c8d8aaf51c59b38680b1be23345"},"cell_type":"code","source":"cols = (feature_importance_df_2[[\"Feature\", \"importance\"]]\n        .groupby(\"Feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df_2.loc[feature_importance_df_2.Feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"Feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"975a59c9819861b0f33fe56cdd889238557c6b63"},"cell_type":"markdown","source":"#### Stacking"},{"metadata":{"trusted":true,"_uuid":"4631c2b589fc492e53790f5ff493b3f466f4bc51"},"cell_type":"code","source":"from sklearn.linear_model import BayesianRidge\n\ntrain_stack = np.vstack([oof,oof_2]).transpose()\ntest_stack = np.vstack([predictions, predictions_2]).transpose()\n\nfolds_stack = RepeatedKFold(n_splits=5, n_repeats=1, random_state=4590)\noof_stack = np.zeros(train_stack.shape[0])\npredictions_3 = np.zeros(test_stack.shape[0])\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds_stack.split(train_stack,target)):\n    print(\"fold {}\".format(fold_))\n    trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values\n    val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values\n    \n    clf_3 = BayesianRidge()\n    clf_3.fit(trn_data, trn_y)\n    \n    oof_stack[val_idx] = clf_3.predict(val_data)\n    predictions_3 += clf_3.predict(test_stack) / 5\n    \nnp.sqrt(mean_squared_error(target.values, oof_stack))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9aa9b3ad41cf77cbe80fd0cc0d2ab2511c367705"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sample_submission.csv')\nsample_submission['target'] = predictions_3\nsample_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cca1b3616e48fb093dd802307cc9863855656ed2"},"cell_type":"code","source":"sample_submission['target'] = (predictions * 0.125+ predictions_2*0.5+predictions_3*0.325)*1.1\nsample_submission.to_csv(\"sub_2.csv\", index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}