{"cells":[{"metadata":{"_uuid":"1dee50e02ba20b8aabe9775e0f8b19c40e1bcff8"},"cell_type":"markdown","source":"This is a version of Ashish Gupta's notebook, which in turn was based on Chau Ngoc Huynh's kernel (3.699) and Panchajanya Banerjee's idea to include the holidays. This purpose of this notebook is just to save teh feature-engineered train and test sets, so that they can be further used in other kenels. \n\n\nSpecial Brazil holidays:\n\nFor 2017-18\n\n1. Mother's Day : Second Sunday of May : May 13 2018\n2. Father's Day : Second Sunday of August :  August 13 2017\n3. Valentine's Day : 12th June, 2017\n4. Children's Day : 12th October 2017\n5. Black Friday : 24th November 2017\n6. Christmas day : 25th December 2017\n"},{"metadata":{"trusted":true,"_uuid":"a594abe9f66c5a090177dff966d6bedf63718a9c"},"cell_type":"code","source":"# Import the necessary libraries\nimport numpy as np\nimport pandas as pd\nimport os\nimport time\nimport warnings\nimport gc\nimport os\nfrom six.moves import urllib\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nwarnings.filterwarnings('ignore')\n%matplotlib inline\nplt.style.use('seaborn')\nfrom scipy.stats import norm, skew\nfrom sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfbbf30dcb4c354827aae6fff1aeaa632a9e874e"},"cell_type":"code","source":"#Add All the Models Libraries\n\n# Scalers\nfrom sklearn.utils import shuffle\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.pipeline import FeatureUnion\n\n# Models\n\nfrom sklearn.linear_model import Lasso\nfrom sklearn.metrics import mean_squared_log_error,mean_squared_error, r2_score,mean_absolute_error\n\nfrom sklearn.model_selection import train_test_split #training and testing data split\nfrom sklearn import metrics #accuracy measure\nfrom sklearn.metrics import confusion_matrix #for confusion matrix\nfrom scipy.stats import reciprocal, uniform\n\nfrom sklearn.model_selection import StratifiedKFold\n\n# Cross-validation\nfrom sklearn.model_selection import KFold #for K-fold cross validation\nfrom sklearn.model_selection import cross_val_score #score evaluation\nfrom sklearn.model_selection import cross_val_predict #prediction\nfrom sklearn.model_selection import cross_validate\n\n# GridSearchCV\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RandomizedSearchCV\n\n#Common data processors\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nfrom sklearn import feature_selection\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.utils import check_array\nfrom scipy import sparse","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"798008f46ddb84ca5ec248990d56b64e5ec84e5a"},"cell_type":"code","source":"# to make this notebook's output stable across runs\nnp.random.seed(123)\ngc.collect()\n# To plot pretty figures\n%matplotlib inline\nplt.rcParams['axes.labelsize'] = 14\nplt.rcParams['xtick.labelsize'] = 12\nplt.rcParams['ytick.labelsize'] = 12","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47d5d544fa7e4bf9305814409c3cabc148b0169a"},"cell_type":"code","source":"#Reduce the memory usage - Inspired by Panchajanya Banerjee\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"450f05f1bb8bd470972ec8a8f80a96f4b56402a7"},"cell_type":"code","source":"train = reduce_mem_usage(pd.read_csv('../input/train.csv',parse_dates=[\"first_active_month\"]))\ntest = reduce_mem_usage(pd.read_csv('../input/test.csv', parse_dates=[\"first_active_month\"]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2352513f1d53487e789e41fbfa050199154f8916"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78407176c3c40b9d15227165028d6b61f94f3ba3"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce244a5a05abaa8c91ff3c73d8c5f86ba592c9a1"},"cell_type":"code","source":"# Now extract the month, year, day, weekday\ntrain[\"month\"] = train[\"first_active_month\"].dt.month\ntrain[\"year\"] = train[\"first_active_month\"].dt.year\ntrain['week'] = train[\"first_active_month\"].dt.weekofyear\ntrain['dayofweek'] = train['first_active_month'].dt.dayofweek\ntrain['days'] = (datetime.date(2018, 2, 1) - train['first_active_month'].dt.date).dt.days\n\ntest[\"month\"] = test[\"first_active_month\"].dt.month\ntest[\"year\"] = test[\"first_active_month\"].dt.year\ntest['week'] = test[\"first_active_month\"].dt.weekofyear\ntest['dayofweek'] = test['first_active_month'].dt.dayofweek\ntest['days'] = (datetime.date(2018, 2, 1) - test['first_active_month'].dt.date).dt.days","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b54460353af274062f649afe26f034c4126a6a84"},"cell_type":"markdown","source":"Now we will try to extract more features from Transactions Data"},{"metadata":{"trusted":true,"_uuid":"f9835dd6582d3e9c36954b498810929c8438d2d2"},"cell_type":"code","source":"# Taking Reference from Other Kernels\ndef aggregate_transaction_hist(trans, prefix):  \n        \n    agg_func = {\n        'purchase_date' : ['max','min'],\n        'month_diff' : ['mean'],\n        'weekend' : ['sum', 'mean'],\n        'authorized_flag': ['sum', 'mean'],\n        'category_1': ['sum','mean'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n        'installments': ['sum', 'mean', 'max', 'min', 'std'],  \n        #'merchant_id': ['nunique'],\n        'month_lag': ['max','min','mean','var'],\n        'month_diff' : ['mean'],\n        'card_id' : ['size'],\n        'month': ['nunique'],\n        'hour': ['nunique'],\n        'weekofyear': ['nunique'],\n        'dayofweek': ['nunique'],\n        'year': ['nunique'],\n        'subsector_id': ['nunique'],\n        'merchant_category_id' : ['nunique'],\n        'Christmas_Day_2017':['mean'],\n        #'Mothers_Day_2017':['mean'],\n        'fathers_day_2017':['mean'],\n        'Children_day_2017':['mean'],\n        'Black_Friday_2017':['mean'],\n        'Valentine_day_2017':['mean'],\n        'Mothers_Day_2018':['mean']\n    }\n    \n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06a04f3393a83cb2687b76deba417012e18ef606","scrolled":true},"cell_type":"code","source":"transactions = reduce_mem_usage(pd.read_csv('../input/historical_transactions.csv'))\ntransactions['authorized_flag'] = transactions['authorized_flag'].map({'Y': 1, 'N': 0})\ntransactions['category_1'] = transactions['category_1'].map({'Y': 1, 'N': 0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1decf075c9422fe20823691972f055b8a1c84f50","scrolled":true},"cell_type":"code","source":"#Feature Engineering - Adding new features inspired by Chau's first kernel\ntransactions['purchase_date'] = pd.to_datetime(transactions['purchase_date'])\ntransactions['year'] = transactions['purchase_date'].dt.year\ntransactions['weekofyear'] = transactions['purchase_date'].dt.weekofyear\ntransactions['month'] = transactions['purchase_date'].dt.month\ntransactions['dayofweek'] = transactions['purchase_date'].dt.dayofweek\ntransactions['weekend'] = (transactions.purchase_date.dt.weekday >=5).astype(int)\ntransactions['hour'] = transactions['purchase_date'].dt.hour \ntransactions['month_diff'] = ((datetime.datetime.today() - transactions['purchase_date']).dt.days)//30\ntransactions['month_diff'] += transactions['month_lag']\n\n#impute missing values - This is now excluded.\ntransactions['category_2'] = transactions['category_2'].fillna(1.0,inplace=True)\ntransactions['category_3'] = transactions['category_3'].fillna('A',inplace=True)\ntransactions['merchant_id'] = transactions['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93a165f209a8853e4c8e1f624becd0506e6fb0f1"},"cell_type":"code","source":"agg_func = {\n        'mean': ['mean'],\n    }\nfor col in ['category_2','category_3']:\n    transactions[col+'_mean'] = transactions['purchase_amount'].groupby(transactions[col]).agg(agg_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dee055f4b8bbb8be045d9a109aae2299d3b2b242"},"cell_type":"code","source":"# New Features with Key Shopping times considered in the dataset. if the purchase has been made within 60 days, it is considered as an influence\n#Christmas : December 25 2017\ntransactions['Christmas_Day_2017'] = (pd.to_datetime('2017-12-25') - transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#Mothers Day: May 14 2017\n#transactions['Mothers_Day_2017'] = (pd.to_datetime('2017-05-04') - transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#fathers day: August 13 2017\ntransactions['fathers_day_2017'] = (pd.to_datetime('2017-08-13') - transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#Childrens day: October 12 2017\ntransactions['Children_day_2017'] = (pd.to_datetime('2017-10-12') - transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#Black Friday : 24th November 2017\ntransactions['Black_Friday_2017'] = (pd.to_datetime('2017-11-24') - transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#Valentines Day\ntransactions['Valentine_day_2017'] = (pd.to_datetime('2017-06-12') - transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n\n#2018\n#Mothers Day: May 13 2018\ntransactions['Mothers_Day_2018'] = (pd.to_datetime('2018-05-13') - transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"95128e635d5c0ad26b170652971aca4fc9c41ad1"},"cell_type":"code","source":"merge_trans = aggregate_transaction_hist(transactions, prefix='hist_')\ndel transactions\ngc.collect()\ntrain = pd.merge(train, merge_trans, on='card_id',how='left')\ntest = pd.merge(test, merge_trans, on='card_id',how='left')\ndel merge_trans\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2ecbbf35630592774f720ccf88849cb8dfe9987"},"cell_type":"code","source":"train.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3e6c6e3dec6f2153d02a684fb7ddbbace4c5550"},"cell_type":"code","source":"#Feature Engineering - Adding new features inspired by Chau's first kernel\ntrain['hist_purchase_date_max'] = pd.to_datetime(train['hist_purchase_date_max'])\ntrain['hist_purchase_date_min'] = pd.to_datetime(train['hist_purchase_date_min'])\ntrain['hist_purchase_date_diff'] = (train['hist_purchase_date_max'] - train['hist_purchase_date_min']).dt.days\ntrain['hist_purchase_date_average'] = train['hist_purchase_date_diff']/train['hist_card_id_size']\ntrain['hist_purchase_date_uptonow'] = (datetime.datetime.today() - train['hist_purchase_date_max']).dt.days\ntrain['hist_first_buy'] = (train['hist_purchase_date_min'] - train['first_active_month']).dt.days\nfor feature in ['hist_purchase_date_max','hist_purchase_date_min']:\n    train[feature] = train[feature].astype(np.int64) * 1e-9\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"370757bfba37b643f463f385d9e878ff27e320bc"},"cell_type":"code","source":"#Feature Engineering - Adding new features inspired by Chau's first kernel\ntest['hist_purchase_date_max'] = pd.to_datetime(test['hist_purchase_date_max'])\ntest['hist_purchase_date_min'] = pd.to_datetime(test['hist_purchase_date_min'])\ntest['hist_purchase_date_diff'] = (test['hist_purchase_date_max'] - test['hist_purchase_date_min']).dt.days\ntest['hist_purchase_date_average'] = test['hist_purchase_date_diff']/test['hist_card_id_size']\ntest['hist_purchase_date_uptonow'] = (datetime.datetime.today() - test['hist_purchase_date_max']).dt.days\ntest['hist_first_buy'] = (test['hist_purchase_date_min'] - test['first_active_month']).dt.days\nfor feature in ['hist_purchase_date_max','hist_purchase_date_min']:\n    test[feature] = test[feature].astype(np.int64) * 1e-9","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88381a0c6e50924860a133e7d61838c86ee1eab1"},"cell_type":"code","source":"# Taking Reference from Other Kernels\ndef aggregate_transaction_new(trans, prefix):  \n        \n    agg_func = {\n        'purchase_date' : ['max','min'],\n        'month_diff' : ['mean'],\n        'weekend' : ['sum', 'mean'],\n        'authorized_flag': ['sum'],\n        'category_1': ['sum','mean'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n        'installments': ['sum', 'mean', 'max', 'min', 'std'],  \n        #'merchant_id': ['nunique'],\n        'month_lag': ['max','min','mean','var'],\n        'month_diff' : ['mean'],\n        'card_id' : ['size'],\n        'month': ['nunique'],\n        'hour': ['nunique'],\n        'weekofyear': ['nunique'],\n        'dayofweek': ['nunique'],\n        'year': ['nunique'],\n        'subsector_id': ['nunique'],\n        'merchant_category_id' : ['nunique'],\n        'Christmas_Day_2017':['mean'],\n        #'Mothers_Day_2017':['mean'],\n        'fathers_day_2017':['mean'],\n        'Children_day_2017':['mean'],\n        'Black_Friday_2017':['mean'],\n        'Valentine_Day_2017' : ['mean'],\n        'Mothers_Day_2018':['mean']\n    }\n    \n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c653335214ef763a6d9e13de46db5800a75c5a3"},"cell_type":"code","source":"# Now extract the data from the new transactions\nnew_transactions = reduce_mem_usage(pd.read_csv('../input/new_merchant_transactions.csv'))\nnew_transactions['authorized_flag'] = new_transactions['authorized_flag'].map({'Y': 1, 'N': 0})\nnew_transactions['category_1'] = new_transactions['category_1'].map({'Y': 1, 'N': 0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd7fc65f351e45fdd1e8a43b730b8ed50b2e4456","scrolled":false},"cell_type":"code","source":"#Feature Engineering - Adding new features inspired by Chau's first kernel\nnew_transactions['purchase_date'] = pd.to_datetime(new_transactions['purchase_date'])\nnew_transactions['year'] = new_transactions['purchase_date'].dt.year\nnew_transactions['weekofyear'] = new_transactions['purchase_date'].dt.weekofyear\nnew_transactions['month'] = new_transactions['purchase_date'].dt.month\nnew_transactions['dayofweek'] = new_transactions['purchase_date'].dt.dayofweek\nnew_transactions['weekend'] = (new_transactions.purchase_date.dt.weekday >=5).astype(int)\nnew_transactions['hour'] = new_transactions['purchase_date'].dt.hour \nnew_transactions['month_diff'] = ((datetime.datetime.today() - new_transactions['purchase_date']).dt.days)//30\nnew_transactions['month_diff'] += new_transactions['month_lag']\n\n#impute missing values\nnew_transactions['category_2'] = new_transactions['category_2'].fillna(1.0,inplace=True)\nnew_transactions['category_3'] = new_transactions['category_3'].fillna('A',inplace=True)\nnew_transactions['merchant_id'] = new_transactions['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)\n\n# New Features with Key Shopping times considered in the dataset. if the purchase has been made within 60 days, \n# it is considered as an influence\n\n#Christmas : December 25 2017\nnew_transactions['Christmas_Day_2017'] = (pd.to_datetime('2017-12-25') - new_transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#Mothers Day: May 14 2017 - Was not significant in Feature Importance\n#new_transactions['Mothers_Day_2017'] = (pd.to_datetime('2017-06-04') - new_transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#fathers day: August 13 2017\nnew_transactions['fathers_day_2017'] = (pd.to_datetime('2017-08-13') - new_transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#Childrens day: October 12 2017\nnew_transactions['Children_day_2017'] = (pd.to_datetime('2017-10-12') - new_transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#Valentine's Day : 12th June, 2017\nnew_transactions['Valentine_Day_2017'] = (pd.to_datetime('2017-06-12') - new_transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n#Black Friday : 24th November 2017\nnew_transactions['Black_Friday_2017'] = (pd.to_datetime('2017-11-24') - new_transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n\n#2018\n#Mothers Day: May 13 2018\nnew_transactions['Mothers_Day_2018'] = (pd.to_datetime('2018-05-13') - new_transactions['purchase_date']).dt.days.apply(lambda x: x if x > 0 and x < 100 else 0)\n\nagg_func = {\n        'mean': ['mean'],\n    }\nfor col in ['category_2','category_3']:\n    new_transactions[col+'_mean'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg(agg_func)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"5ebf2d2ac4064c8e9a008f555c7f902ca0fdf853"},"cell_type":"code","source":"merge_new = aggregate_transaction_new(new_transactions, prefix='new_')\ndel new_transactions\ngc.collect()\n\ntrain = pd.merge(train, merge_new, on='card_id',how='left')\ntest = pd.merge(test, merge_new, on='card_id',how='left')\ndel merge_new\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4741d24d2c897ebd286807d0d3bd658800e316ca"},"cell_type":"code","source":"#Feature Engineering - Adding new features inspired by Chau's first kernel\ntrain['new_purchase_date_max'] = pd.to_datetime(train['new_purchase_date_max'])\ntrain['new_purchase_date_min'] = pd.to_datetime(train['new_purchase_date_min'])\ntrain['new_purchase_date_diff'] = (train['new_purchase_date_max'] - train['new_purchase_date_min']).dt.days\ntrain['new_purchase_date_average'] = train['new_purchase_date_diff']/train['new_card_id_size']\ntrain['new_purchase_date_uptonow'] = (datetime.datetime.today() - train['new_purchase_date_max']).dt.days\ntrain['new_first_buy'] = (train['new_purchase_date_min'] - train['first_active_month']).dt.days\nfor feature in ['new_purchase_date_max','new_purchase_date_min']:\n    train[feature] = train[feature].astype(np.int64) * 1e-9\n\n#Feature Engineering - Adding new features inspired by Chau's first kernel\ntest['new_purchase_date_max'] = pd.to_datetime(test['new_purchase_date_max'])\ntest['new_purchase_date_min'] = pd.to_datetime(test['new_purchase_date_min'])\ntest['new_purchase_date_diff'] = (test['new_purchase_date_max'] - test['new_purchase_date_min']).dt.days\ntest['new_purchase_date_average'] = test['new_purchase_date_diff']/test['new_card_id_size']\ntest['new_purchase_date_uptonow'] = (datetime.datetime.today() - test['new_purchase_date_max']).dt.days\ntest['new_first_buy'] = (test['new_purchase_date_min'] - test['first_active_month']).dt.days\nfor feature in ['new_purchase_date_max','new_purchase_date_min']:\n    test[feature] = test[feature].astype(np.int64) * 1e-9\n    \n#added new feature - Interactive\ntrain['card_id_total'] = train['new_card_id_size'] + train['hist_card_id_size']\ntrain['purchase_amount_total'] = train['new_purchase_amount_sum'] + train['hist_purchase_amount_sum']\n\ntest['card_id_total'] = test['new_card_id_size'] + test['hist_card_id_size']\ntest['purchase_amount_total'] = test['new_purchase_amount_sum'] + test['hist_purchase_amount_sum']\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de5dbc4519c349552d66f192c47d73872ddf0c71","scrolled":true},"cell_type":"code","source":"# Now check the shape of Train and Test Data\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d19048974a748af94a1db2a98f754dad9934f8be"},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e98856ac9c9f94acc517b8008f9dcb273228bbf2","scrolled":true},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26324d62739c824e870c7f97d247c2744899d422"},"cell_type":"markdown","source":"Imputations and Data Transformation"},{"metadata":{"trusted":true,"_uuid":"b3dd37511e6a80f0ce02ec552a9930bbb3a5b309","scrolled":true},"cell_type":"code","source":"#Check for missing values in training set\nnulls = np.sum(train.isnull())\nnullcols = nulls.loc[(nulls != 0)]\ndtypes = train.dtypes\ndtypes2 = dtypes.loc[(nulls != 0)]\ninfo = pd.concat([nullcols, dtypes2], axis=1).sort_values(by=0, ascending=False)\n#print(info)\n#print(\"There are\", len(nullcols), \"columns with missing values in data set\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1737d40d9a033b7de10b4e401743ee1f2324fa32"},"cell_type":"code","source":"#Check for missing values in training set\nnulls = np.sum(test.isnull())\nnullcols = nulls.loc[(nulls != 0)]\ndtypes = test.dtypes\ndtypes2 = dtypes.loc[(nulls != 0)]\ninfo = pd.concat([nullcols, dtypes2], axis=1).sort_values(by=0, ascending=False)\n#print(info)\n#print(\"There are\", len(nullcols), \"columns with missing values in test set\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1b45687c9755b0672753aa84a0765480ff1d67b"},"cell_type":"markdown","source":"Impute any values will significantly affect the RMSE score for test set. So Imputations have been excluded"},{"metadata":{"trusted":true,"_uuid":"868b8f492ded29822f14d74fb4350f03ee4b3f2d"},"cell_type":"code","source":"numeric_dtypes = ['float64']\nnumerics = []\nfor i in train.columns:\n    if train[i].dtype in numeric_dtypes: \n        numerics.append(i)\n        \n#train.update(train[numerics].fillna(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd0491f1ce6813e21979ca54e534bfe5bdd76260"},"cell_type":"code","source":"numeric_dtypes = ['float64']\nnumerics = []\nfor i in test.columns:\n    if test[i].dtype in numeric_dtypes: \n        numerics.append(i)\n        \n#test.update(test[numerics].fillna(0))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b0f12459d09a2d88ea34a60c85c1aa7553ef1aa"},"cell_type":"markdown","source":"Detect and Correct Outliers"},{"metadata":{"trusted":true,"_uuid":"0dd1c98207ed527cfd85a6b1a71235f3424629ee"},"cell_type":"code","source":"# Remove the Outliers if any \ntrain['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\ntrain['outliers'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b5d796a748bbdf483888c96f37634f07e4c8663"},"cell_type":"code","source":"for features in ['feature_1','feature_2','feature_3']:\n    order_label = train.groupby([features])['outliers'].mean()\n    train[features] = train[features].map(order_label)\n    test[features] =  test[features].map(order_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff17dbfc2d4087d08c9ee9a719c6f4f867692b48"},"cell_type":"code","source":"# Get the X and Y\ndf_train_columns = [c for c in train.columns if c not in ['card_id', 'first_active_month', 'outliers']]\ndf_test_columns = [c for c in train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5a2945056e41f7c612821d8d5be552a7d944de1"},"cell_type":"code","source":"train['fold'] = 0\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1974)\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train,train['outliers'].values)):\n    train['fold'].iloc[val_idx] = fold_\n    \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75b60f1f4a5de71fdfc081e952b57f54c9658e8a"},"cell_type":"code","source":"df_train_columns.append('fold')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2aa124e3731a2e6e010f3558b33cecc85c05af1e"},"cell_type":"code","source":"train.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a0ed2a5b0ab317cf8697fbdcd18f780859d6299"},"cell_type":"code","source":"train[df_train_columns].to_csv('new_train.csv', index=False)\ntest[df_test_columns].to_csv('new_test.csv', index=False)\ntrain[['card_id', 'outliers']].to_csv('outliers', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}