{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Elo Merchant Category Recommendation :

\n\nNote :
\nBaseline draft for submission is Kaggle Notebook ver_58 with best scores updated
\n(https://github.com/samaujs/Data-Science/blob/main/Kaggle/sample/elo-eda-ml.ipynb)
\n\n> Rank : 329 out of 4110 [Public Score : 3.68346] (8.00%) with Kaggle Notebook ver_57 (130 KSfeatures_opt2_blending)
\n> Rank : 352 out of 4110 [Private Score : 3.61319] (8.56%) with Kaggle Notebook ver_31 (142 features)
","metadata":{"id":"udn5diOPRIHf"}},{"cell_type":"code","source":"! nvidia-smi -L","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Import all the libraries :**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\n# %matplotlib inline\n# %config InlineBackend.figure_format = 'retina'\n# plt.style.use('ggplot')\n\nfrom scipy.stats import ks_2samp\n\nimport datetime\nimport seaborn as sns\nsns.set_style(\"whitegrid\")\nimport warnings \nwarnings.simplefilter(\"ignore\")\n\n# For computing Variable Inflation Factor (VIF)\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\n\n# For memory garbage collection\nimport gc\n# Use joblib instead of pickle to save a model in scikit-learn\nimport pickle\nimport joblib\n\n# Display columns\npd.set_option('display.max_rows', 210)\npd.set_option('display.max_columns', 210)\npd.set_option('display.max_info_columns', 210)\nprint(pd.get_option(\"display.max_rows\"), pd.get_option(\"display.max_columns\"))\nprint(pd.get_option(\"display.max_info_rows\"), pd.get_option(\"display.max_info_columns\"))","metadata":{"id":"a0aazA45RIHm","execution":{"iopub.status.busy":"2021-11-09T10:24:11.483287Z","iopub.execute_input":"2021-11-09T10:24:11.483700Z","iopub.status.idle":"2021-11-09T10:24:12.446539Z","shell.execute_reply.started":"2021-11-09T10:24:11.483607Z","shell.execute_reply":"2021-11-09T10:24:12.445174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Functions :**","metadata":{}},{"cell_type":"code","source":"## Reference: https://www.kaggle.com/rinnqd/reduce-memory-usage\ndef reduce_memory_usage(df, verbose=True):\n '''\n This function reduces the memory sizes of dataframe by changing the datatypes of the columns.\n Parameters\n df - DataFrame whose size to be reduced\n verbose - Boolean, to mention the verbose required or not.\n '''\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2\n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64)\n else:\n c_prec = df[col].apply(lambda x: np.finfo(x).precision).max()\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max and c_prec == np.finfo(np.float16).precision:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max and c_prec == np.finfo(np.float32).precision:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n end_mem = df.memory_usage().sum() / 1024**2\n if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df\n\n# Check for missing values : train_data['new_hist_first_buy'].isna().any()\ndef check_missing_values(df):\n cols_missing_values = []\n for col in df.columns:\n if df[col].isna().any():\n cols_missing_values.append(col)\n print(col)\n return cols_missing_values\n \ndef create_new_columns(name, aggs):\n # get the individual key from dictionary and the corresponding list of functions for this key\n # For example : 'purchase_amount' key for functions ['sum','max','min','mean','var']\n return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]\n\n# Intersection function\ndef intersection(list1, list2): \n return list(set(list1) & set(list2))\n\n# Load saved model\ndef load_model_from_picklefile(filename):\n infile = open(filename,'rb')\n loaded_model = pickle.load(infile)\n infile.close()\n return loaded_model\n\n# Save trained model\ndef save_model_to_picklefile(filename, save_model):\n model_file = open(filename,'wb')\n pickle.dump(save_model, model_file)\n model_file.close()\n\n# Create file for submission to Kaggle\ndef create_file_for_submission(filename, card_ids, final_predictions): \n kaggle = pd.DataFrame({'card_id': card_ids, 'target': final_predictions})\n kaggle.to_csv(filename, index=False)","metadata":{"id":"Tiiadbd6DATL","execution":{"iopub.status.busy":"2021-11-09T10:24:12.448170Z","iopub.execute_input":"2021-11-09T10:24:12.448487Z","iopub.status.idle":"2021-11-09T10:24:12.471453Z","shell.execute_reply.started":"2021-11-09T10:24:12.448457Z","shell.execute_reply":"2021-11-09T10:24:12.470274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.models import Word2Vec\n\ndef word2vec_feature(prefix, df, groupby, target, size):\n df_bag = pd.DataFrame(df[[groupby, target]])\n df_bag[target] = df_bag[target].astype(str)\n df_bag[target].fillna('NAN', inplace=True)\n df_bag = df_bag.groupby(groupby, as_index=False)[target].agg({'list':(lambda x: list(x))}).reset_index()\n doc_list = list(df_bag['list'].values)\n \n w2v = Word2Vec(doc_list, vector_size=size, window=3, min_count=1, workers=32)\n vocab_keys = list(w2v.wv.index_to_key) # w2v.wv.vocab.keys() \n w2v_array = []\n for v in vocab_keys :\n w2v_array.append(list(w2v.wv[v]))\n df_w2v = pd.DataFrame()\n df_w2v['vocab_keys'] = vocab_keys \n df_w2v = pd.concat([df_w2v, pd.DataFrame(w2v_array)], axis=1)\n df_w2v.columns = [target] + ['w2v_%s_%s_%d'%(prefix,target,x) for x in range(size)]\n print ('df_w2v:' + str(df_w2v.shape))\n return df_w2v","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Load data (Wall time: 1min 47s)\ntrain_data = pd.read_csv('../input/elo-merchant-category-recommendation/train.csv')\ntest_data = pd.read_csv('../input/elo-merchant-category-recommendation/test.csv')\nhistorical_data = pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv')\nnewmerchant_data = pd.read_csv('../input/elo-merchant-category-recommendation/new_merchant_transactions.csv')\nmerchants_data = pd.read_csv('../input/elo-merchant-category-recommendation/merchants.csv')","metadata":{"id":"1DjRBpmES6g3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Reduce memory usage of data (Wall time: 4min 25s)\ntrain_data = reduce_memory_usage(train_data)\ntest_data = reduce_memory_usage(test_data)\nhistorical_data = reduce_memory_usage(historical_data)\nnewmerchant_data = reduce_memory_usage(newmerchant_data)\nmerchants_data = reduce_memory_usage(merchants_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape, test_data.shape, historical_data.shape, newmerchant_data.shape, merchants_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 1 : Building Machine Learning models","metadata":{}},{"cell_type":"markdown","source":"# **(A) Data Preprocessing and Feature Engineering (Exploratory Data Analysis)**\n> #### With the performed EDA (refer to Kaggle Notebook version 2 for complete analysis), the following data manipulations can be carried out for building the machine learning models for the task.\n> Eg. We observe from that the train and test datasets have similar distributions.\n> They are all categorical variables\n> feature_1, feature_2 and feature_3 has 5, 3 and 2 (binary) unique values respectively.","metadata":{}},{"cell_type":"markdown","source":"### Examining the feature statistics :","metadata":{}},{"cell_type":"code","source":"# EDA : Check for numeric and non-numeric features\n# numeric_feature_count = 0\n# non_numeric_feature_count = 0\n# numeric_features = []\n# non_numeric_features = []\n\n# for col in historical_data.columns:\n# if (historical_data[col].dtypes == 'object'):\n# non_numeric_feature_count += 1\n# non_numeric_features.append(col)\n# print(\"Non-numeric feature No. {} and name : {}\".format(non_numeric_feature_count, col))\n# print(\"No. of Missing values : {}, Zero values : {} with Mode value : {}\".\\\n# format(historical_data[col].isnull().sum(), (historical_data[col] == 0).sum(),\n# historical_data[col].mode().values[0]))\n# print(\"*\" * 82)\n# print()\n# else:\n# numeric_feature_count += 1\n# numeric_features.append(col)\n# print(\"Numeric feature No. {} and name : {}\".format(numeric_feature_count, col))\n# # (historical_data[col] == historical_data[col].median()).sum()\n# print(\"No. of Missing values : {}, Zero values : {} with Mode value : {}\".\\\n# format(historical_data[col].isnull().sum(), (historical_data[col] == 0).sum(),\n# historical_data[col].mode().values[0])) # Get only the value without the index\n# print(\"-\" * 82)\n# print()\n \n\n# print(\"Total No. of Numeric features {} and Non-numeric features : {}\".format(numeric_feature_count, non_numeric_feature_count))\n# print(\"\\nNumeric features : {} and \\nNon-numeric features : {}\".format(numeric_features, non_numeric_features))","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EDA : There are multiple transactions a single 'card_id'\n# for card_id[0], 'C_ID_0ab67a22ab' : there are 1304310 - 1304243 = 67 historical transactions\n# for card_id[10], 'C_ID_4859ac9ed5' : there are 23622204 - 23622180 = 24 historical transactions\n# historical_data.loc[23622180:23622204]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check for missing values in the loaded data frames","metadata":{}},{"cell_type":"code","source":"check_missing_values(train_data), check_missing_values(test_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_missing_values(historical_data), check_missing_values(newmerchant_data), check_missing_values(merchants_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Impute missing values (with mean, median, mode) :\n* For historical and new merchant data with mode values","metadata":{}},{"cell_type":"code","source":"# historical_data['category_2'].value_counts()\n# historical_data['category_2'].unique() = [1., nan, 3., 5., 2., 4.]\nhistorical_data_cat2_mode = historical_data['category_2'].mode()\nhistorical_data_merchant_id_mode = historical_data['merchant_id'].mode()\n# historical_data['category_3'].value_counts()\nhistorical_data_cat3_mode = historical_data['category_3'].mode()\n\nprint(\"historical_data mode for category_2 : {}\".format(historical_data_cat2_mode[0]))\nprint(\"historical_data mode for merchant_id : {}\".format(historical_data_merchant_id_mode[0]))\nprint(\"historical_data mode for category_3 : {}\".format(historical_data_cat3_mode[0]))\n\n# Replace missing values with mode values for category_3', 'merchant_id' and 'category_2'\n# When inplace = True, the data is modified in place,\n# which means it will return nothing and the dataframe is now updated.\nfor df in [historical_data, newmerchant_data]:\n df['category_2'].fillna(historical_data_cat2_mode[0], inplace = True)\n df['merchant_id'].fillna(historical_data_merchant_id_mode[0], inplace = True)\n df['category_3'].fillna(historical_data_cat3_mode[0], inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observe outliers and replaced with median values in provided features :\n* For historical and new merchant data with mode values","metadata":{}},{"cell_type":"code","source":"# print('Quantile values for purchase amount in Historical Transaction :')\n# print('25th Percentile :',historical_data['purchase_amount'].quantile(0.25))\n# print('50th Percentile :',historical_data['purchase_amount'].quantile(0.50))\n# print('75th Percentile :',historical_data['purchase_amount'].quantile(0.75))\n# print('100th Percentile :',historical_data['purchase_amount'].quantile(1))\n# print('\\n******************************************************************\\n')\n# print('Quantile values for purchase amount in New Merchant Transaction :')\n# print('25th Percentile :',newmerchant_data['purchase_amount'].quantile(0.25))\n# print('50th Percentile :',newmerchant_data['purchase_amount'].quantile(0.50))\n# print('75th Percentile :',newmerchant_data['purchase_amount'].quantile(0.75))\n# print('100th Percentile :',newmerchant_data['purchase_amount'].quantile(1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# historical_data[historical_data['purchase_amount'] == 6010603.9717525]\nhistorical_data_outlier_index = historical_data.loc[(historical_data['purchase_amount'] == 6010603.9717525)].index.values\nprint(\"The index of purchase amount in historical data :\", historical_data_outlier_index)\n\n# newmerchant_data[newmerchant_data['purchase_amount'] == 263.15749789]\nnewmerchant_data_outlier_index = newmerchant_data.loc[(newmerchant_data['purchase_amount'] == 263.15749789), 'purchase_amount'].index.values\nprint(\"The index of purchase amount in newmerchant data :\", newmerchant_data_outlier_index)\n\nprint(\"Original 'Purchase amount' in historical data :\", historical_data.loc[historical_data_outlier_index[0], 'purchase_amount'])\nprint(\"Original 'Purchase amount' in newmerchant data :\", newmerchant_data.loc[newmerchant_data_outlier_index[0], 'purchase_amount'])\n\n# Replace outlier 'purchase amount' with median in historical data\nhistorical_data.loc[historical_data_outlier_index[0], 'purchase_amount'] = historical_data['purchase_amount'].median()\n# Replace outlier 'purchase amount' with median in newmerchant data\nnewmerchant_data.loc[newmerchant_data_outlier_index[0], 'purchase_amount'] = newmerchant_data['purchase_amount'].median()\n\nprint(\"New median 'Purchase amount' in historical data :\", historical_data.loc[historical_data_outlier_index[0], 'purchase_amount'])\nprint(\"New median 'Purchase amount' in newmerchant data :\", newmerchant_data.loc[newmerchant_data_outlier_index[0], 'purchase_amount'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Impute missing value in test data :\n* Replace with the earliest purchase date","metadata":{}},{"cell_type":"code","source":"# test_data['first_active_month'].isna().value_counts()\nmissing_card_index = test_data.loc[test_data['first_active_month'].isna(), 'card_id'].index.values\n\nprint(\"The index of missing card in test data :\", missing_card_index)\nprint(\"The missing cards in test data :\\n\", test_data.loc[test_data['first_active_month'].isna()])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# missing_card_id = test_data.loc[test_data['first_active_month'].isna(), 'card_id'].reset_index(drop=True)[0]\n# get the historical data for the missing card_id\n# card_missing_first_active_month = historical_data.loc[historical_data['card_id'] == missing_card_id]\n\n# print(\"Card_id : {} with {} transactions.\".format(missing_card_id,\n# card_missing_first_active_month.shape[0]))\n\n# Replace with Earliest 'purchase_date' for card_id 'C_ID_c27b4f80f7' is '2017-03-09' or \n# Fill the missing value with the mode value '2017-09' in 'first_active_month' of test_data\n# 55 transactions with card_id = 'C_ID_c27b4f80f7' that is without first_active_month\n\n# format : YYYY-MM\ntest_data['first_active_month'].fillna('2017-03',inplace=True)\n\n# test_data.loc[missing_card_index[0]]['first_active_month']\ntest_data.loc[test_data['card_id'] == 'C_ID_c27b4f80f7'] # 2017-09","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_missing_values(train_data), check_missing_values(test_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* # Feature Engineering\n* Aggregations (Historical, New Merchant and Merchant data)\n* Feature Transformations (with mappings, data type, mean)\n* Feature Engineering with feature creations and subset selections (from 4 to 151)","metadata":{}},{"cell_type":"code","source":"# The purchase dates in historical_data from 2017-01-01 to 2018-02-28\n# The purchase dates in newmerchant_data from 2017-03-01 to 2018-04-30\n\n# historical_data['installments'].median() is 0.0\n# newmerchant_data['installments'].median() is 1.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Generalization for 'purchase date' in 'historical data' and 'new merchant data'","metadata":{}},{"cell_type":"code","source":"for df in [historical_data, newmerchant_data]:\n df_purchase_date_max = pd.to_datetime(df['purchase_date'].max())\n \n df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n df['category_1'] = df['category_1'].map({'Y':1, 'N':0}) \n #https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73244\n \n # map category improves in ranking\n df['category_3'] = df['category_3'].map({'A':2, 'B':1, 'C':0})\n \n # 7 new columns for historical_data and newmerchant_data\n df['purchase_date_year'] = df['purchase_date'].dt.year\n df['purchase_date_weekofyear'] = df['purchase_date'].dt.weekofyear\n df['purchase_date_month'] = df['purchase_date'].dt.month\n df['purchase_date_dayofweek'] = df['purchase_date'].dt.dayofweek\n df['purchase_date_weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n df['purchase_date_hour'] = df['purchase_date'].dt.hour\n \n # Replace \"datetime.datetime.today()\" and apply floor division; which improves on the ranking\n df['purchase_date_month_diff'] = ((df_purchase_date_max - df['purchase_date']).dt.days)//30\n df['purchase_date_month_diff'] += df['month_lag']\n \n # Explore purchase_amount_sum_mean in 'category_1' which helps to improve on ranking\n # df['category_1_purchase_amount_sum'] = df.groupby(['category_1'])['purchase_amount'].transform('sum') # 'sum'\n # df['category_1_purchase_amount_mean'] = df.groupby(['category_1'])['purchase_amount'].transform('mean') # 'sum'\n \n # Replace 'installments' outliers and later replaced collectively with median for missing values\n df['installments'].replace(-1, np.nan, inplace=True) # 0.0 and 1.0 for historical_data and newmerchant_data respectively\n df['installments'].replace(999, np.nan, inplace=True)\n\nhistorical_data['purchase_date'].head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional 7 new columns for historical_data and newmerchant_data\nhistorical_data.shape, newmerchant_data.shape, train_data.shape, test_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_missing_values(historical_data), check_missing_values(newmerchant_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create features with aggregations for 'purchase_amount'\n- These are memory intensive operations; thus needs to be done before Word2Vec","metadata":{}},{"cell_type":"code","source":"# Aggregation done within loop below will cause notebook to restart\n# historical_data\n# hist_sum_category_1_1_purchase_amount\ntarget = 'purchase_amount'\ncol = 'category_1'\nfor cat1_unique_val in historical_data[col].unique():\n print(\"Creating {} column with category value {} with sum of {}\".format(col, cat1_unique_val, target))\n print('card_sum_' + str(col) + '_' + str(cat1_unique_val) + '_' + str(target))\n print(\"-\" * 80)\n historical_data['card_sum_' + str(col) + '_' + str(cat1_unique_val) + '_' + str(target)] =\\\n historical_data['card_id'].map(historical_data[historical_data[col]==cat1_unique_val].groupby(['card_id'])[target].sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# historical_data\n# authorized_flag + purchase_amount\ncol = 'authorized_flag'\nfor authflag_unique_val in historical_data[col].unique():\n print(\"Creating {} column with authorized_flag value {} with sum of {}\".format(col, authflag_unique_val, target))\n print('card_sum_' + str(col) + '_' + str(authflag_unique_val) + '_' + str(target))\n print(\"-\" * 80)\n historical_data['card_sum_' + str(col) + '_' + str(authflag_unique_val) + '_' + str(target)] =\\\n historical_data['card_id'].map(historical_data[historical_data[col]==authflag_unique_val].groupby(['card_id'])[target].sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = 'category_1'\n# newmerchant_data\n# category_1 + purchase_amount\nfor cat1_unique_val in newmerchant_data[col].unique():\n print(\"Creating {} column with category value {} with sum of {}\".format(col, cat1_unique_val, target))\n print('card_sum_' + str(col) + '_' + str(cat1_unique_val) + '_' + str(target))\n print(\"-\" * 80)\n newmerchant_data['card_sum_' + str(col) + '_' + str(cat1_unique_val) + '_' + str(target)] =\\\n newmerchant_data['card_id'].map(newmerchant_data[newmerchant_data[col]==cat1_unique_val].groupby(['card_id'])[target].sum())\n\nprint(\"Shapes of historical_data and newmerchant_data after {} :\".format(str(col) + '_' + str(target)))\nprint(historical_data.shape, newmerchant_data.shape)\n\n# KS discard features 'card_sum_category_1_0_purchase_amount' later","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# newmerchant_data\n# month_lag + purchase_amount\n# col = 'month_lag'\n\n# KS discard features 'card_sum_month_lag_1_purchase_amount', 'card_sum_month_lag_2_purchase_amount'\n# for mthlag_unique_val in newmerchant_data[col].unique():\n# print(\"Creating {} column with month_lag value {} with sum of {}\".format(col, mthlag_unique_val, target))\n# print('card_sum_' + str(col) + '_' + str(mthlag_unique_val) + '_' + str(target))\n# print(\"-\" * 80)\n# newmerchant_data['card_sum_' + str(col) + '_' + str(mthlag_unique_val) + '_' + str(target)] =\\\n# newmerchant_data['card_id'].map(newmerchant_data[newmerchant_data[col]==mthlag_unique_val].groupby(['card_id'])[target].sum()) \n \n# print(\"Shapes of historical_data and newmerchant_data after {} :\".format(str(col) + '_' + str(target)))\n# print(historical_data.shape, newmerchant_data.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = 'authorized_flag'\n# newmerchant_data\n# authorized_flag + purchase_amount\nfor authflag_unique_val in newmerchant_data[col].unique():\n print(\"Creating {} column with authorized_flag value {} with sum of {}\".format(col, authflag_unique_val, target))\n print('card_sum_' + str(col) + '_' + str(authflag_unique_val) + '_' + str(target))\n print(\"-\" * 80)\n newmerchant_data['card_sum_' + str(col) + '_' + str(authflag_unique_val) + '_' + str(target)] =\\\n newmerchant_data['card_id'].map(newmerchant_data[newmerchant_data[col]==authflag_unique_val].groupby(['card_id'])[target].sum())\n\nprint(\"Shapes of historical_data and newmerchant_data after {} :\".format(str(col) + '_' + str(target)))\nprint(historical_data.shape, newmerchant_data.shape)\n\n# KS discard features 'card_sum_authorized_flag_1_purchase_amount' later","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* # Word2Vec\n- 'merchant_id' and 'merchant_category_id' in 'historical_data'\n- 'merchant_category_id', 'merchant_id', 'city_id', 'state_id' and 'subsector_id' in 'newmerchant_data'","metadata":{}},{"cell_type":"code","source":"%%time\n# Wall time: 6min 2s\nhist_merchant_id_w2v = word2vec_feature('hist', historical_data, 'card_id', 'merchant_id', 5) # category_3\nhist_merchant_category_id_w2v = word2vec_feature('hist', historical_data, 'card_id', 'merchant_category_id', 5) # category_3\n# hist_city_id_w2v = word2vec_feature('hist', historical_data, 'card_id', 'city_id', 5) # category_3\n\n# w2v values vary in different runs\nhist_merchant_id_w2v","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting hist_merchant_category_id_w2v['merchant_category_id'] type from 'object' into 'int16'\nhist_merchant_category_id_w2v['merchant_category_id'] = hist_merchant_category_id_w2v['merchant_category_id'].astype(np.int16)\n# hist_city_id_w2v['city_id'] = hist_city_id_w2v['city_id'].astype(np.int16)\n\n# hist_city_id_w2v['city_id'].dtype\nhist_merchant_category_id_w2v['merchant_category_id'].dtype","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merging is memory-intensive with word2vec\nhistorical_data = historical_data.merge(hist_merchant_category_id_w2v, on='merchant_category_id', how='left')\nhistorical_data = historical_data.merge(hist_merchant_id_w2v, on='merchant_id', how='left')\nhistorical_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Data type of hist_merchant_id_w2v['merchant_id'] :\", hist_merchant_id_w2v['merchant_id'].dtype)\ndel hist_merchant_id_w2v, hist_merchant_category_id_w2v # hist_city_id_w2v\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# newmerchant_data[['merchant_category_id','merchant_id','city_id','state_id','subsector_id']].nunique()\n# newmerchant_data[['merchant_category_id','merchant_id','city_id','state_id','subsector_id']].head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Wall time: 2min 11s\n# KS discard features\n# new_merchant_city_id_w2v = word2vec_feature('new', newmerchant_data, 'card_id', 'city_id', 5)\n# new_merchant_subsector_id_w2v = word2vec_feature('new', newmerchant_data, 'card_id', 'subsector_id', 5)\n# new_merchant_id_w2v = word2vec_feature('new', newmerchant_data, 'card_id', 'merchant_id', 5) # category_3\n# new_merchant_category_id_w2v = word2vec_feature('new', newmerchant_data, 'card_id', 'merchant_category_id', 5)\nnew_merchant_state_id_w2v = word2vec_feature('new', newmerchant_data, 'card_id', 'state_id', 5) # category_3\n\n# w2v values vary in different runs\nnew_merchant_state_id_w2v","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_merchant_id_w2v['merchant_id'] = new_merchant_id_w2v['merchant_id'].astype(np.int16)\n# newmerchant_data['merchant_id'].head(2)\n\n# new_merchant_id_w2v['merchant_id'].dtype, new_merchant_category_id_w2v['merchant_category_id'].dtype,\\\n# new_merchant_city_id_w2v['city_id'].dtype, new_merchant_state_id_w2v['state_id'].dtype,\\\n# new_merchant_subsector_id_w2v['subsector_id'].dtype\nhistorical_data.shape, newmerchant_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_merchant_city_id_w2v['city_id'] = new_merchant_city_id_w2v['city_id'].astype(np.int16)\n# new_merchant_subsector_id_w2v['subsector_id'] = new_merchant_subsector_id_w2v['subsector_id'].astype(np.int16)\n# new_merchant_id_w2v['merchant_id'] = new_merchant_id_w2v['merchant_id'].astype(str)\n# new_merchant_category_id_w2v['merchant_category_id'] = new_merchant_category_id_w2v['merchant_category_id'].astype(np.int16)\nnew_merchant_state_id_w2v['state_id'] = new_merchant_state_id_w2v['state_id'].astype(np.int16)\n\n# new_merchant_id_w2v['merchant_id'].dtype, new_merchant_category_id_w2v['merchant_category_id'].dtype,\\\n# new_merchant_city_id_w2v['city_id'].dtype, new_merchant_subsector_id_w2v['subsector_id'].dtype, \\\nnew_merchant_state_id_w2v['state_id'].dtype","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# newmerchant_data = newmerchant_data.merge(new_merchant_city_id_w2v, on='city_id', how='left')\n# newmerchant_data = newmerchant_data.merge(new_merchant_subsector_id_w2v, on='subsector_id', how='left')\n# newmerchant_data = newmerchant_data.merge(new_merchant_id_w2v, on='merchant_id', how='left')\n# newmerchant_data = newmerchant_data.merge(new_merchant_category_id_w2v, on='merchant_category_id', how='left')\nnewmerchant_data = newmerchant_data.merge(new_merchant_state_id_w2v, on='state_id', how='left')\n\nhistorical_data.shape, newmerchant_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del new_merchant_city_id_w2v, new_merchant_subsector_id_w2v, new_merchant_id_w2v, new_merchant_category_id_w2v\ndel new_merchant_state_id_w2v\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregate 'historical_data' with 'card_id'","metadata":{}},{"cell_type":"code","source":"historical_data['month_lag'].value_counts()\n# historical_data['category_1'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\naggs = {}\n\n# 'dayofweek', 'year'\nfor col in ['purchase_date_month', 'purchase_date_hour', 'purchase_date_weekofyear', 'subsector_id', 'merchant_id']: # 'merchant_category_id', 'month','hour','weekofyear'\n # 'state_id', 'city_id']: # added more features\n aggs[col] = ['nunique']\n\n# KS discard feature 'hist_purchase_amount_max', keep to create other ratio features that are useful\naggs['purchase_amount'] = ['sum', 'min', 'max', 'mean', 'std', 'median'] # check 'std' instead of var'\n# KS discard feature 'hist_installments_mean'\naggs['installments'] = ['sum', 'max', 'mean', 'std'] # 'median', 'min', 'mean', check 'std' instead of var'\naggs['purchase_date'] = ['min', 'max'] # peak-to-peak (maximum - minimum)\naggs['month_lag'] = ['min', 'mean', 'std'] # ['max','min','mean','var'], check 'std' instead of var'\naggs['authorized_flag'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\naggs['purchase_date_weekend'] = ['sum', 'mean']\naggs['purchase_date_month_diff'] = ['mean']\n\n# Check features if can achieve better performance\n# These features did not help to ascend the rank climb\n# aggs['purchase_date_year'] = ['nunique', 'sum', 'mean']\n# aggs['purchase_date_weekofyear'] = ['nunique', 'sum', 'mean']\n# aggs['purchase_date_month'] = ['nunique', 'sum', 'mean']\n# aggs['purchase_date_dayofweek'] = ['nunique', 'sum', 'mean']\n# aggs['purchase_date_hour'] = ['nunique', 'sum', 'mean']\n\n# Check features if can achieve better performance\naggs['merchant_category_id'] = ['nunique', 'std']\n\n# Can try by simply make a copy without aggregations\naggs['card_sum_category_1_0_purchase_amount'] = ['sum'] # 'std', 'mean'\naggs['card_sum_category_1_1_purchase_amount'] = ['sum'] # 'std', 'mean'\n\n# 'hist_card_sum_authorized_flag_0_purchase_amount_mean'\naggs['card_sum_authorized_flag_0_purchase_amount'] = ['sum'] # 'std', 'mean'\naggs['card_sum_authorized_flag_1_purchase_amount'] = ['mean', 'sum'] # 'std'\n\naggs['w2v_hist_merchant_category_id_0'] = ['std']\naggs['w2v_hist_merchant_category_id_1'] = ['std']\naggs['w2v_hist_merchant_category_id_2'] = ['std']\naggs['w2v_hist_merchant_category_id_3'] = ['std']\naggs['w2v_hist_merchant_category_id_4'] = ['std']\n\naggs['w2v_hist_merchant_id_0'] = ['std']\naggs['w2v_hist_merchant_id_1'] = ['std']\n# KS discard features\n# 'hist_w2v_hist_merchant_id_2_std', 'hist_w2v_hist_merchant_id_3_std'\naggs['w2v_hist_merchant_id_2'] = ['std']\naggs['w2v_hist_merchant_id_3'] = ['std']\naggs['w2v_hist_merchant_id_4'] = ['std']\n\n\n# Explore purchase_amount_sum_mean\n# aggs['category_1_purchase_amount_sum'] = ['mean', 'sum']\n# aggs['category_1_purchase_amount_mean'] = ['mean', 'sum']\n\n# added more features with median for 'installments' and ptp for 'purchase_date'\n# aggs['month'] = [ 'min', 'max', 'mean', 'var']\n\n# KS discard features\n# 'hist_category_2_purchase_amount_mean_mean', 'hist_category_3_purchase_amount_mean_mean',\n# 'hist_category_3_purchase_amount_sum_mean',\n# Create 'purchase_amount_mean' for the groupby columns\n# Include 'category_1' for groupby sum\nfor col in ['category_1', 'category_2', 'category_3']:\n historical_data[col+'_sum_purchase_amount'] = historical_data.groupby([col])['purchase_amount'].transform('sum') # 'mean'\n # aggs[col+'_purchase_amount_mean'] = ['mean']\n aggs[col+'_sum_purchase_amount'] = ['sum', 'mean', 'std']\n\n\n# Eg: 'hist_month_nunique', 'hist_hour_nunique'\nnew_columns = create_new_columns('hist',aggs)\nprint(\"New aggregated columns :\\n\", new_columns)\n\n# Perform the aggregations with 'card_id'\ndf_hist_trans_group = historical_data.groupby('card_id').agg(aggs)\n\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\n\nprint(\"Constructing features without aggregations ...\")\n# Create features without aggregations \ndf_hist_trans_group['hist_purchase_date_diff'] = (df_hist_trans_group['hist_purchase_date_max'] - df_hist_trans_group['hist_purchase_date_min']).dt.days\ndf_hist_trans_group['hist_purchase_date_average'] = df_hist_trans_group['hist_purchase_date_diff']/df_hist_trans_group['hist_card_id_size']\ndf_hist_trans_group['hist_purchase_date_uptonow'] = (datetime.datetime.today() - df_hist_trans_group['hist_purchase_date_max']).dt.days\n\n# Explore new features for rank ascend\ndf_hist_trans_group['hist_card_last_woy'] = df_hist_trans_group['hist_purchase_date_max'].dt.weekofyear\ndf_hist_trans_group['hist_card_last_doy'] = df_hist_trans_group['hist_purchase_date_max'].dt.dayofyear\ndf_hist_trans_group['hist_card_last_day'] = df_hist_trans_group['hist_purchase_date_max'].dt.day\n\n# --\nprint(\"-\" * 80)\n# References :\n# https://www.kaggle.com/raddar/target-true-meaning-revealed\n# https://www.kaggle.com/raddar/towards-de-anonymizing-the-data-some-insights\ntarget = 'hist_purchase_amount_new'\nprint(\"\\nConstructing : 'hist_card_sum_purchase_amount_new'\")\ndf_hist_trans_group['hist_purchase_amount_new'] = np.round((historical_data['purchase_amount'] / 0.00150265118) + 497.06, 2)\n# Checkout new feature for effects on RMSE\ndf_hist_trans_group['hist_card_sum_purchase_amount_new'] = df_hist_trans_group['card_id'].map(df_hist_trans_group.groupby(['card_id'])[target].sum())\ndf_hist_trans_group['hist_card_mean_purchase_amount_new'] = df_hist_trans_group['card_id'].map(df_hist_trans_group.groupby(['card_id'])[target].mean())\ndf_hist_trans_group['hist_card_median_purchase_amount_new'] = df_hist_trans_group['card_id'].map(df_hist_trans_group.groupby(['card_id'])[target].median())\ndf_hist_trans_group['hist_card_min_purchase_amount_new'] = df_hist_trans_group['card_id'].map(df_hist_trans_group.groupby(['card_id'])[target].min())\ndf_hist_trans_group['hist_card_max_purchase_amount_new'] = df_hist_trans_group['card_id'].map(df_hist_trans_group.groupby(['card_id'])[target].max())\ndf_hist_trans_group['hist_card_std_purchase_amount_new'] = df_hist_trans_group['card_id'].map(df_hist_trans_group.groupby(['card_id'])[target].std())\n\n# 'hist_card_sum_month_lag_0_purchase_amount_new'\n# target = 'hist_card_sum_purchase_amount_new'\n# print(\"Constructing : 'hist_card_sum_month_lag_x_purchase_amount_new'\")\n# df_hist_trans_group['hist_month_lag'] = historical_data['month_lag']\n# KS discard feature 'hist_card_sum_month_lag_0_purchase_amount_new', 'hist_card_sum_month_lag_neg2_purchase_amount_new'\n# Replace df_hist_trans_group with historical_data will cause insufficient memories in the notebook\n# df_hist_trans_group['hist_card_sum_month_lag_neg2_purchase_amount_new'] = \\\n# df_hist_trans_group['card_id'].map(df_hist_trans_group[df_hist_trans_group['hist_month_lag']==-2].groupby(['card_id'])[target].sum())\n\n# Check which variation has higher importance ranking\nprint(\"Copying : variations of 'category_1'\")\n# card_sum_category_1_x_purchase_amount\ndf_hist_trans_group['hist_card_sum_category_1_0_purchase_amount'] = historical_data['card_sum_category_1_0_purchase_amount']\n\n# KS discard feature 'hist_card_sum_category_1_1_purchase_amount'\ndf_hist_trans_group['hist_card_sum_category_1_1_purchase_amount'] = historical_data['card_sum_category_1_1_purchase_amount']\n# 'hist_sum_category_1_1_purchase_amount' generated above with separate columns of aggregrations\ndf_hist_trans_group['hist_sum_category_1_purchase_amount'] = historical_data['category_1_sum_purchase_amount']\n\n# These features did not help to ascend the rank climb\ntarget = 'purchase_amount'\nprint(\"Constructing : 'hist_sum_category_1_1_purchase_amount'\")\ndf_hist_trans_group['hist_category_1'] = historical_data['category_1']\ndf_hist_trans_group['purchase_amount'] = historical_data['purchase_amount']\ndf_hist_trans_group['hist_sum_category_1_1_purchase_amount'] = \\\n df_hist_trans_group['hist_category_1'].map(df_hist_trans_group[df_hist_trans_group['hist_category_1']==1].groupby(['hist_category_1'])[target].sum())\n\n# Explore new building feature, 'card_purchase_date_max_ratio', for rank ascend\ndf_hist_trans_group['hist_purchase_date_int'] = historical_data['purchase_date'].astype(np.int64) * 1e-9\ndf_hist_trans_group['hist_card_purchase_date_max'] = df_hist_trans_group.groupby(['card_id'])['hist_purchase_date_int'].transform('max')\n# --\n\nprint()\nprint(\"Shape of df_hist_trans_group :\\n\", df_hist_trans_group.shape)\nprint(\"df_hist_trans_group columns :\\n\", df_hist_trans_group.columns)\n\n\ndf_train = train_data.merge(df_hist_trans_group, on='card_id',how='left')\ndf_test = test_data.merge(df_hist_trans_group, on='card_id',how='left')\n\n# (325540, 60) in v50 to (325540, 76) in v54\n# Features with low importances\n# 'hist_sum_category_1_1_purchase_amount','hist_card_mean_purchase_amount_new',\n# 'hist_card_sum_purchase_amount_new', 'hist_card_min_purchase_amount_new',\n# 'hist_category_1', 'hist_sum_category_1_purchase_amount', \n# 'new_merchant_card_min_purchase_amount_new', 'new_merchant_card_sum_purchase_amount_new',\n# 'new_merchant_purchase_amount_new', 'new_merchant_card_median_purchase_amount_new'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# historical_data['month_lag'].nunique() gives 14\n# historical_data (out of memory), could try within loop above by just copying the needed column\n# month_lag + purchase_amount\n# col = 'month_lag'\n\n# for mthlag_unique_val in historical_data[col].unique():\n# print(\"Creating {} column with month_lag value {} with sum of {}\".format(col, mthlag_unique_val, target))\n# print('card_sum_' + str(col) + '_' + str(mthlag_unique_val) + '_' + str(target))\n# print(\"-\" * 80)\n# historical_data['card_sum_' + str(col) + '_' + str(mthlag_unique_val) + '_' + str(target)] =\\\n# historical_data['card_id'].map(historical_data[historical_data[col]==mthlag_unique_val].groupby(['card_id'])[target].sum()) \n \n# print(\"Shapes of historical_data and newmerchant_data after {} :\".format(str(col) + '_' + str(target)))\n# print(historical_data.shape, newmerchant_data.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After merging with historical data with 'left' with group by card_id\nprint(df_hist_trans_group.shape, df_train.shape, df_test.shape)\n\n# Release memory\ndel df_hist_trans_group, train_data, test_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create more aggregated features from 'hist_purchase_amount_new'","metadata":{}},{"cell_type":"code","source":"# Check for missing values\n# train_data['new_hist_first_buy'].isna().any()\n# check_missing_values(df_train), check_missing_values(df_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ((201917, 50), (123623, 49))\ndf_train.shape, df_test.shape, historical_data.shape, newmerchant_data.shape\n# df_train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Exploring New Merchant data","metadata":{}},{"cell_type":"code","source":"# historical_data['category_1_purchase_amount_sum'], newmerchant_data['category_1_purchase_amount_sum']\n# check_missing_values(newmerchant_data), check_missing_values(merchants_data)\n\n# There are no duplicate card_ids in train_data with 201,917 transactions\n# train_data_cards = train_data['card_id']\n# train_data_cards.nunique()\n\n# test_data_cards = test_data['card_id']\n# There are no duplicate card_ids in test_data with 123,623 transactions\n# test_data_cards.nunique()\n\n# print(\"Total number of unique cards in train_data and test_data :\", (train_data_cards.nunique() + test_data_cards.nunique()))\n\n# historical_data_cards = historical_data['card_id']\n# The unique card_ids in train_data and test_date makes up the unique 325,540 card_ids in historical_data\n# historical_data_cards.nunique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Note :\n> There are NO intersected values between train and test sets\n\n> All unique card_ids in train_data and test_data can be found in historical_data\n\n> All unique card_ids in train_data and test_data can be found in newmerchant_data","metadata":{}},{"cell_type":"code","source":"# Total number of unique cards in train_data and test_data = Total of unique cards in historical_data\n# intersect_list = intersection(train_data_cards, test_data_cards)\n# print(\"There are NO intersected card_id :\", intersect_list)\n\n# np.intersect1d(train_data_cards, test_data_cards)\n# intersect_list = list(set(train_data_cards) & set(test_data_cards))\n# print(\"No. of intersected card_ids :\", len(intersect_list))\n# --\n# pandas.core.series.Series\n# s1 = pd.Series([4, 5, 20, 42, 42, 43])\n# s2 = pd.Series([1, 2, 3, 5, 42])\n# set(s1)&set(s2) same results\n# np.intersect1d(s1, s2)\n# --\n# Difference in card_ids\n# card_difference = set(train_data_cards).symmetric_difference(test_data_cards)\n# print(\"Total no. of card_ids in train_data and test_data which are unique : {}\".format(len(card_difference)))\n# --\n# union_cards_df = pd.concat([train_data_cards, test_data_cards])\n\n# type(union_cards_df) is pandas.core.series.Series\n# union_data_size = train_data_cards.nunique() + test_data_cards.nunique() = 325,540\n# The card_ids in historical_data intersects with all the card_ids found in train and test sets\n# intersect_list = intersection(historical_data_cards, union_cards_df)\n# len(intersect_list)\n# --\n# card_difference = set(historical_data_cards).symmetric_difference(union_cards_df)\n# print(\"The card_ids in train and test but NOT found in historical_data : {}\".format(len(card_difference)))\n# --\n# union_data_size = train_data_cards.nunique() + test_data_cards.nunique() = 325,540\n# Union returns a new set with elements from the set and the specified iterables\n# card_union = set(train_data_cards).union(test_data_cards)\n# print(\"Total no. of unique cards : \", len(card_union))\n\n# symmetric_difference returns a new set with elements in either the set or the specified iterable but not both.\n# The card_ids in historical_data can be found in train and test sets; there are no difference in card_ids\n# card_difference = set(historical_data_cards).symmetric_difference(union_cards_df)\n# card_difference = card_union.symmetric_difference(historical_data_cards)\n# print(\"Unique cards of train and test data but not in historical data : \", len(card_difference))\n# --\n# newmerchant_data_cards = newmerchant_data['card_id']\n# There are 290,001 unique cards in newmerchant_data but train_data has 201,917 cards\n# print(\"No. of unique cards in train_data :\", train_data_cards.nunique())\n# print(\"No. of unique cards in newmerchant_data :\", newmerchant_data_cards.nunique())\n\n# intersect_list_train_newmerchant = intersection(train_data_cards, newmerchant_data_cards)\n# print(\"No. of intersected card_ids between train_data_cards and newmerchant_data_cards :\", len(intersect_list_train_newmerchant))\n\n# These cards create NaNs during merging\n# print(\"Unique card_ids in train_data but not in the newmerchant_data : \",\n# train_data_cards.nunique() - len(intersect_list_train_newmerchant))\n# --\n# card_difference = set(train_data_cards).symmetric_difference(newmerchant_data_cards)\n# print(\"Unique card_ids not in the intersection of train_data and the newmerchant_data : \", len(card_difference))\n# --\n# There are 290,001 unique cards in newmerchant_data but train_data has 201,917 cards\n# print(\"No. of unique cards in test_data :\", test_data_cards.nunique())\n# print(\"No. of unique cards in newmerchant_data :\", newmerchant_data_cards.nunique())\n\n# intersect_list_test_newmerchant = intersection(test_data_cards, newmerchant_data_cards)\n# print(\"No. of intersected card_ids between test_data_cards and newmerchant_data_cards :\", len(intersect_list_test_newmerchant))\n\n# These cards create NaNs during merging\n# print(\"Unique card_ids in test_data but not in the newmerchant_data : \",\n# test_data_cards.nunique() - len(intersect_list_test_newmerchant))\n# --\n# train_test_newmerchant_card_union = set(intersect_list_train_newmerchant).union(intersect_list_test_newmerchant)\n# len(intersect_list_train_newmerchant) + len(intersect_list_test_newmerchant) gives same result of 290,001\n# len(train_test_newmerchant_card_union)\n# --\n# card_difference = train_test_newmerchant_card_union.symmetric_difference(newmerchant_data_cards)\n# print(\"Verify that intersected unique cards of train and test data with newmerchant data but not in newmerchant data : \",\n# len(card_difference))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Exploring Merchant data","metadata":{}},{"cell_type":"code","source":"merchants_data_ids = merchants_data['merchant_id']\nnewmerchants_data_ids = newmerchant_data['merchant_id']\n\n# There are 334,633 merchant_id in merchants_data but newmerchant_data has only 226,129 merchant_id\nprint(\"No. of unique merchant_id in merchants_data :\", merchants_data_ids.nunique())\nprint(\"No. of unique merchant_id in newmerchant_data :\", newmerchants_data_ids.nunique())\n\nintersect_list_newmerchants_merchants = intersection(newmerchants_data_ids, merchants_data_ids)\nprint(\"No. of intersected merchant_ids between newmerchants_data_ids and merchants_data_ids :\", len(intersect_list_newmerchants_merchants))\n\n# All merchant_id in newmerchant_data can be found in merchants_data\nprint(\"Unique merchant_ids in newmerchants_data but not in the merchants_data :\",\n newmerchants_data_ids.nunique() - len(intersect_list_newmerchants_merchants))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merchants_data_ids.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merchants_data_cat_ids = merchants_data['merchant_category_id']\nnewmerchants_data_cat_ids = newmerchant_data['merchant_category_id']\n\n# There are 334,633 merchant_id in merchants_data but newmerchant_data has only 226,129 merchant_id\nprint(\"No. of unique merchant_category_id in merchants_data :\", merchants_data_cat_ids.nunique())\nprint(\"No. of unique merchant_category_id in newmerchant_data :\", newmerchants_data_cat_ids.nunique())\n\nintersect_list_newmerchants_merchants_cat_id = intersection(newmerchants_data_cat_ids, merchants_data_cat_ids)\nprint(\"No. of intersected merchant_category_ids between newmerchants_data_cat_ids and merchants_data_cat_ids :\",\n len(intersect_list_newmerchants_merchants_cat_id))\n\n# These merchant_category_id create NaNs during merging\nprint(\"Unique merchant_category_ids in newmerchants_data but not in the merchants_data :\",\n newmerchants_data_cat_ids.nunique() - len(intersect_list_newmerchants_merchants_cat_id))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merchants_data_ids = merchants_data['merchant_id']\nhistorical_data_merchant_ids = historical_data['merchant_id']\n\n# There are 334,633 merchant_id in merchants_data but newmerchant_data has only 226,129 merchant_id\nprint(\"No. of unique merchant_id in merchants_data :\", merchants_data_ids.nunique())\nprint(\"No. of unique merchant_id in historical_data :\", historical_data_merchant_ids.nunique())\n\nintersect_list_historical_merchants = intersection(historical_data_merchant_ids, merchants_data_ids)\nprint(\"No. of intersected merchant_ids between historical_data_merchant_ids and merchants_data_ids :\",\n len(intersect_list_historical_merchants))\n\n# All merchant_id in historical_data can be found in merchants_data\nprint(\"Unique merchant_ids in historical_data but not in the merchants_data :\",\n historical_data_merchant_ids.nunique() - len(intersect_list_historical_merchants))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merchants_data_cat_ids = merchants_data['merchant_category_id']\nhistorical_data_cat_ids = historical_data['merchant_category_id']\n\n# There are 334,633 merchant_id in merchants_data but newmerchant_data has only 226,129 merchant_id\nprint(\"No. of unique merchant_category_id in merchants_data :\", merchants_data_cat_ids.nunique())\nprint(\"No. of unique merchant_category_id in historical_data :\", historical_data_cat_ids.nunique())\n\nintersect_list_historical_merchants_cat_id = intersection(historical_data_cat_ids, merchants_data_cat_ids)\nprint(\"No. of intersected merchant_category_ids between historical_data_cat_ids and merchants_data_cat_ids :\",\n len(intersect_list_historical_merchants_cat_id))\n\n# These merchant_category_id create NaNs during merging\nprint(\"Unique merchant_category_ids in historical_data but not in the merchants_data :\",\n historical_data_cat_ids.nunique() - len(intersect_list_historical_merchants_cat_id))\n\nprint()\nprint(\"There are MORE merchant_category_ids in historical_data than in the merchants_data :\",\n (historical_data_cat_ids.nunique() - merchants_data_cat_ids.nunique()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merchants_data['merchant_id'].describe()\n# df_train.shape, df_test.shape, newmerchant_data.shape, merchants_data.shape\n# check_missing_values(newmerchant_data), check_missing_values(merchants_data)\n\n# There are duplicate values of 'merchant_id'\n# merchants_data['merchant_id'].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Impute missing values in merchants data with respective mode values","metadata":{}},{"cell_type":"code","source":"# print(merchants_data['avg_sales_lag3'].isna().value_counts())\n# print()\n# print(\"Mode value of avg_sales_lag3 :\", merchants_data['avg_sales_lag3'].mode()[0])\n\n# missing_avg_sales_l3_index = merchants_data.loc[merchants_data['avg_sales_lag3'].isna(), 'merchant_id'].index.values\n# print(\"The index of missing avg_sales_lag3 in merchants_data :\", missing_avg_sales_l3_index)\n\n# merchants_data.loc[merchants_data['avg_sales_lag3'].isna()]\n# --\n# print(merchants_data['avg_sales_lag6'].isna().value_counts())\n# print()\n# print(\"Mode value of avg_sales_lag6 :\", merchants_data['avg_sales_lag6'].mode()[0])\n\n# missing_avg_sales_l6_index = merchants_data.loc[merchants_data['avg_sales_lag6'].isna(), 'merchant_id'].index.values\n# print(\"The index of missing avg_sales_lag6 in merchants_data :\", missing_avg_sales_l6_index)\n\n# merchants_data.loc[merchants_data['avg_sales_lag6'].isna()]\n# --\n# print(merchants_data['avg_sales_lag12'].isna().value_counts())\n# print()\n# print(\"Mode value of avg_sales_lag12 :\", merchants_data['avg_sales_lag12'].mode()[0])\n\n# missing_avg_sales_l12_index = merchants_data.loc[merchants_data['avg_sales_lag12'].isna(), 'merchant_id'].index.values\n# print(\"The index of missing avg_sales_lag12 in merchants_data :\", missing_avg_sales_l12_index)\n\n# merchants_data.loc[merchants_data['avg_sales_lag12'].isna()]\n# --\n# print(merchants_data['category_2'].isna().value_counts())\n# print()\n# print(\"Mode value of category_2 :\", merchants_data['category_2'].mode()[0])\n\n# missing_category_2_index = merchants_data.loc[merchants_data['category_2'].isna(), 'merchant_id'].index.values\n# print(\"The index of missing category_2 in merchants_data :\", missing_category_2_index)\n\n# merchants_data.loc[missing_category_2_index]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill missing values in merchants_data with respective mode values\nmerchants_data['avg_sales_lag3'].fillna(merchants_data['avg_sales_lag3'].mode()[0], inplace = True)\nmerchants_data['avg_sales_lag6'].fillna(merchants_data['avg_sales_lag6'].mode()[0], inplace = True)\nmerchants_data['avg_sales_lag12'].fillna(merchants_data['avg_sales_lag12'].mode()[0], inplace = True)\nmerchants_data['category_2'].fillna(merchants_data['category_2'].mode()[0], inplace = True)\nmerchants_data.loc[3:15]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_missing_values(merchants_data), merchants_data.shape, newmerchant_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merging strategy :\n> 1. Aggregate historical data with ***'card_id'*** and merge to train and test with groupby ***'card_id'***\n> 2. Aggregate merchant data with ***'merchant_id'*** and merge to new merchant data with groupby ***'merchant_id'***\n> 3. Merge new merchant data with merchant data to new train and test with groupby ***'card_id'***","metadata":{}},{"cell_type":"code","source":"# merchants_data['most_recent_sales_range'].value_counts()\n\n# merchants_data['most_recent_purchases_range'].value_counts()\n# merchants_data['most_recent_purchases_range'].min(), merchants_data['most_recent_purchases_range'].max()\n\n# merchants_data['most_recent_sales_range'].mode()\n\n# aggs = {}\n# aggs['most_recent_sales_range'] = ['min']\n# md_tmp = merchants_data.groupby('merchant_id').agg(aggs)\n# md_tmp.shape\n\n# md_tmp = pd.DataFrame()\n# Group based on 'purchase_amount'\n# for col in ['most_recent_sales_range','most_recent_purchases_range']:\n# md_tmp[col+'_avg_sales_lag3_mean'] = merchants_data.groupby([col])['avg_sales_lag3'].transform('min')\n# md_tmp[col+'_avg_sales_lag6_mean'] = merchants_data.groupby([col])['avg_sales_lag6'].transform('min')\n# md_tmp[col+'_avg_sales_lag12_mean'] = merchants_data.groupby([col])['avg_sales_lag12'].transform('min')\n\n# md_tmp.reset_index(drop=False,inplace=True)\n# md_tmp.head(2)\n# check_missing_values(md_tmp)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature mappings in merchants_data","metadata":{}},{"cell_type":"code","source":"sales_purchases_range_dict= {'A':5, 'B':4, 'C':3, 'D':2, 'E':1}\n\nfor df in [merchants_data]:\n df['most_recent_sales_range'] = df['most_recent_sales_range'].map(sales_purchases_range_dict)\n df['most_recent_purchases_range'] = df['most_recent_purchases_range'].map(sales_purchases_range_dict)\n # df['category_1'] = df['category_1'].map({'Y':1, 'N':0})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Majority revenue and quantity of transactions in last active month are high in merchants data\n# df['most_recent_sales_range'].value_counts(), df['most_recent_purchases_range'].value_counts(), df['category_1'].value_counts()\n\n# merchants_data['merchant_id'].nunique()\n\n# merchants_data['city_id'].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregate 'merchants_data' with 'merchant_id'","metadata":{}},{"cell_type":"code","source":"aggs = {}\n\nfor col in ['subsector_id', 'category_4','city_id', 'state_id']: # 'merchant_id', 'merchant_group_id'\n aggs[col] = ['nunique']\n\naggs['numerical_1'] = ['sum', 'mean']\n# KS discard feature 'merchants_numerical_2_sum'\naggs['numerical_2'] = ['mean'] # 'sum'\naggs['avg_sales_lag3'] = ['sum', 'mean', 'size', 'min', 'max']\n# KS discard feature 'merchants_avg_purchases_lag3_min'\naggs['avg_purchases_lag3'] = ['sum', 'mean', 'size', 'max'] # 'min'\naggs['active_months_lag3'] = ['sum', 'mean', 'size', 'min', 'max']\naggs['avg_sales_lag6'] = ['sum', 'mean', 'size', 'min', 'max']\n# KS discard feature 'merchants_avg_purchases_lag6_sum'\naggs['avg_purchases_lag6'] = ['mean', 'size', 'min', 'max'] # 'sum'\naggs['active_months_lag6'] = ['sum', 'mean', 'size', 'min', 'max']\naggs['avg_sales_lag12'] = ['sum', 'mean', 'size', 'min', 'max']\n# KS discard feature 'merchants_avg_purchases_lag12_sum', 'merchants_avg_purchases_lag12_min'\naggs['avg_purchases_lag12'] = ['mean', 'size', 'max'] # 'sum', 'min'\naggs['active_months_lag12'] = ['sum', 'mean', 'size', 'min', 'max']\naggs['category_2'] = ['mean']\n\n# Explore more features\naggs['most_recent_sales_range'] = ['min', 'max'] \naggs['most_recent_purchases_range'] = ['min', 'max']\n\n# Removed for low feaature importance\n# aggs['merchant_category_id'] = ['std']\n\n# aggs['merchant_category_id'] = ['sum', 'mean', 'size', 'min', 'max']\n\n# 'category_1_nunique' is not created even if added above with the other col\n# aggs['category_1'] = ['min', 'max', 'nunique']\n\nnew_columns = create_new_columns('merchants',aggs)\nprint(\"New columns :\\n\", new_columns)\ndf_merchant_id_group = merchants_data.groupby('merchant_id').agg(aggs) # merchant_category_id\n\ndf_merchant_id_group.columns = new_columns\ndf_merchant_id_group.reset_index(drop=False, inplace=True)\n\nprint(\"Shape of df_merchant_category :\\n\", df_merchant_id_group.shape)\nprint(\"df_merchant_id_group columns :\\n\", df_merchant_id_group.columns)\ndf_merchant_id_group\n\n# Remove aggs['merchant_category_id'] = ['std'] in v54","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are missing values in 'merchants_merchant_category_id_std'\n# df_merchant_id_group['merchants_merchant_category_id_std'].isna().value_counts()\n# True 334592\n# False 41","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# hist_merchant_id_w2v = word2vec_feature('hist', historical_data, 'card_id', 'merchant_id', 5) # category_3\n# hist_merchant_id_w2v\n# --\n# %%time\n# hist_merchant_category_id_w2v = word2vec_feature('hist', historical_data, 'card_id', 'merchant_category_id', 5) # category_3\n# hist_merchant_category_id_w2v\n# --\n# merchants_data.shape, historical_data.nunique()\n# --\n# merchants_data = merchants_data.merge(hist_merchant_id_w2v, on='merchant_id', how='left') # df_merchant_id_group\n# --\n# # Converting hist_merchant_category_id_w2v['merchant_category_id'] type from 'object' into 'int16'\n# hist_merchant_category_id_w2v['merchant_category_id'] = hist_merchant_category_id_w2v['merchant_category_id'].astype(np.int16)\n# hist_merchant_category_id_w2v['merchant_category_id'].dtype\n# --\n# merchants_data = merchants_data.merge(hist_merchant_category_id_w2v, on='merchant_category_id', how='left')\n# merchants_data.shape\n# --","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group newmerchant_data and merchants_data\nnewmerchant_with_merchants_data = newmerchant_data.merge(df_merchant_id_group, on='merchant_id',how='left') # merchant_category_id\nnewmerchant_with_merchants_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_test.shape, newmerchant_data.shape, merchants_data.shape, df_merchant_id_group.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### When merging with 'merchant_category_id', there are infinite values that should be replaced","metadata":{}},{"cell_type":"code","source":"# np.isinf(df_merchant_id_group).any()\n# np.isinf(df_merchant_id_group).any().value_counts()\n\n# print(\"printing column name where infinity is present\")\n# col_name = df_merchant_id_group.columns.to_series()[np.isinf(df_merchant_id_group).any()]\n# print(col_name)\n\n# print(df_merchant_id_group['merchants_avg_purchases_lag3_sum'].median())\n# print(df_merchant_id_group['merchants_avg_purchases_lag3_mean'].median())\n# print(df_merchant_id_group['merchants_avg_purchases_lag6_sum'].median())\n# print(df_merchant_id_group['merchants_avg_purchases_lag6_mean'].median())\n# print(df_merchant_id_group['merchants_avg_purchases_lag12_sum'].median())\n# print(df_merchant_id_group['merchants_avg_purchases_lag12_mean'].median())\n\n# missing_value_index = df_merchant_id_group.loc[np.isinf(df_merchant_id_group['merchants_avg_purchases_lag3_sum']),\n# 'merchants_avg_purchases_lag3_sum'].index.values\n\n# missing_value_index\n\n# df_merchant_id_group.loc[missing_value_index, 'merchants_avg_purchases_lag3_sum']\n\n# Replace all infinite values to NaN which will replaced with median values collectively\n# df_merchant_id_group.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merchants_data['merchant_id'].describe()\n\n# newmerchant_with_merchants_data.columns\n\n# merchants_data['merchant_category_id'].nunique(), newmerchant_data['merchant_category_id'].nunique()\n\n# newmerchant_with_merchants_data.head(2)\n\n# check_missing_values(newmerchant_with_merchants_data), newmerchant_with_merchants_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Define the aggregation procedure outside of the groupby operation\n# aggregations = {\n# 'purchase_amount': ['sum', 'mean', 'std', 'min', 'max', 'size', 'median']\n# }\n\n# grouped = newmerchant_data.groupby('card_id').agg(aggregations)\n# grouped.columns = grouped.columns.droplevel(level=0)\n# grouped.rename(columns={\n# \"sum\": \"sum_purchase_amount\", \n# \"mean\": \"mean_purchase_amount\",\n# \"std\": \"std_purchase_amount\", \n# \"min\": \"min_purchase_amount\",\n# \"max\": \"max_purchase_amount\", \n# \"size\": \"num_purchase_amount\",\n# \"median\": \"median_purchase_amount\"\n# }, inplace=True)\n# grouped.reset_index(inplace=True)\n\n# grouped.shape\n\n# df_train = pd.merge(df_train, grouped, on=\"card_id\", how=\"left\")\n# df_test = pd.merge(df_test, grouped, on=\"card_id\", how=\"left\")\n\n# del grouped\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Due to card_ids not found in newmerchant_data\n# check_missing_values(df_train), check_missing_values(df_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# newmerchant_with_merchants_data.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merging with 'newmerchant_data' with 'card_id'","metadata":{}},{"cell_type":"code","source":"newmerchant_data['month_lag'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\naggs = {}\n\n# 'year', 'month', 'weekofyear', 'dayofweek', 'hour', 'merchant_id', 'subsector_id',\n# for col in ['merchant_category_id']:\n# # 'city_id', 'state_id']: # added more features\n# aggs[col] = ['nunique']\n\n# KS discard features 'purchase_amount' ['sum', 'mean', 'median']\naggs['purchase_amount'] = ['min', 'max'] # 'std'\naggs['installments'] = ['sum', 'max'] # ['min','mean','var'], ['std', 'median']\naggs['purchase_date'] = ['min', 'max']\naggs['month_lag'] = ['mean'] # ['max','min','mean','var'], 'std'\n\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\n# aggs['weekend'] = ['mean'] # 'sum',\naggs['purchase_date_weekend'] = ['mean'] # 'sum',\n# aggs['month_diff'] = ['mean']\naggs['purchase_date_month_diff'] = ['mean'] # want to try 'sum'\n\n# Check features if can achieve better performance; want to try\n# Some of the aggregations do not improve the overall RMSE from 130 KS features\n# aggs['purchase_date_year'] = ['nunique', 'sum', 'mean']\n# aggs['purchase_date_weekofyear'] = ['nunique', 'sum'] # 'mean'\n# aggs['purchase_date_month'] = ['nunique', 'sum', 'mean']\n# aggs['purchase_date_dayofweek'] = ['nunique', 'sum', 'mean']\n# aggs['purchase_date_hour'] = ['nunique', 'mean'] # 'sum'\n\n# purchase amount aggregations\n# KS discard features 'card_sum_category_1_0_purchase_amount_mean', 'card_sum_category_1_0_purchase_amount_sum'\n# aggs['card_sum_category_1_0_purchase_amount'] = ['mean', 'sum'] # 'std'\n# KS discard features 'card_sum_category_1_1_purchase_amount_mean'\naggs['card_sum_category_1_1_purchase_amount'] = ['sum'] # 'std', 'mean'\n# KS discard features 'card_sum_month_lag_1_purchase_amount_sum', 'card_sum_month_lag_1_purchase_amount_mean'\n# aggs['card_sum_month_lag_1_purchase_amount'] = ['mean', 'sum'] # 'std'\n# KS discard features 'card_sum_month_lag_2_purchase_amount_sum', 'card_sum_month_lag_2_purchase_amount_mean'\n# aggs['card_sum_month_lag_2_purchase_amount'] = ['mean', 'sum'] # 'std'\n\n# 'card_sum_authorized_flag_1_purchase_amount_sum', 'card_sum_authorized_flag_1_purchase_amount_mean'\n# aggs['card_sum_authorized_flag_1_purchase_amount'] = ['mean', 'sum'] # 'std'\n\n# word2vec\n# KS discard features\n# aggs['w2v_new_city_id_0'] = ['std']\n# aggs['w2v_new_city_id_1'] = ['std']\n# aggs['w2v_new_city_id_2'] = ['std']\n# aggs['w2v_new_city_id_3'] = ['std']\n# aggs['w2v_new_city_id_4'] = ['std']\n\n# aggs['w2v_new_subsector_id_0'] = ['std']\n# aggs['w2v_new_subsector_id_1'] = ['std']\n# aggs['w2v_new_subsector_id_2'] = ['std']\n# aggs['w2v_new_subsector_id_3'] = ['std']\n# aggs['w2v_new_subsector_id_4'] = ['std']\n\n# aggs['w2v_new_merchant_id_0'] = ['std']\n# aggs['w2v_new_merchant_id_1'] = ['std']\n# aggs['w2v_new_merchant_id_2'] = ['std']\n# aggs['w2v_new_merchant_id_3'] = ['std']\n# aggs['w2v_new_merchant_id_4'] = ['std']\n\n# aggs['w2v_new_merchant_category_id_0'] = ['std']\n# aggs['w2v_new_merchant_category_id_1'] = ['std']\n# aggs['w2v_new_merchant_category_id_2'] = ['std']\n# aggs['w2v_new_merchant_category_id_3'] = ['std']\n# aggs['w2v_new_merchant_category_id_4'] = ['std']\n\naggs['w2v_new_state_id_0'] = ['std']\naggs['w2v_new_state_id_1'] = ['std']\naggs['w2v_new_state_id_2'] = ['std']\naggs['w2v_new_state_id_3'] = ['std']\naggs['w2v_new_state_id_4'] = ['std']\n\n## To be continue for further feature explorations\n# Check features if can achieve better performance\n# aggs['merchant_id'] = ['std'] # 'nunique' gives error\n# aggs['merchant_category_id'] = ['std'] # 'nunique' gives error\n# Explore category_1_purchase_amount_sum_mean\n# aggs['category_1_purchase_amount_sum'] = ['mean', 'sum']\n# aggs['category_1_purchase_amount_mean'] = ['mean', 'sum']\n\n# aggs['merchants_most_recent_sales_range_min'] = ['min', 'max']\naggs['merchants_most_recent_sales_range_max'] = ['min'] # 'max'\naggs['merchants_most_recent_purchases_range_min'] = ['min', 'max']\naggs['merchants_most_recent_purchases_range_max'] = ['min'] # 'max'\n\naggs['merchants_category_2_mean'] = ['sum'] # , 'min', 'max'\n\n# aggs['merchants_merchant_category_id_sum'] = ['sum', 'mean', 'min', 'max'] # 'size'\n# aggs['merchants_merchant_category_id_mean'] = ['sum', 'mean', 'min', 'max'] # 'size'\n# aggs['merchants_merchant_category_id_size'] = ['sum', 'mean'] # 'size', 'min', 'max'\n# aggs['merchants_merchant_category_id_min'] = ['sum', 'mean', 'min', 'max'] # 'size'\n# aggs['merchants_merchant_category_id_max'] = ['sum', 'mean'] # 'size', 'min', 'max'\n\n# Explore merchants_category_1\n# aggs['merchants_category_1_min'] = ['min', 'max', 'sum']\n# aggs['merchants_category_1_max'] = ['sum'] # 'min', 'max'\n# aggs['merchants_category_1_nunique'] = ['min', 'max']\n##\n\n# include columns with merchant information\n# aggs['merchants_merchant_group_id_nunique'] = ['size']\n# aggs['merchants_subsector_id_nunique'] = ['size']\n# aggs['merchants_category_1_nunique'] = ['size']\n# aggs['merchants_most_recent_sales_range_nunique'] = ['size']\n# aggs['merchants_most_recent_purchases_range_nunique'] = ['size']\n# aggs['merchants_category_4_nunique'] = ['size']\n# aggs['merchants_city_id_nunique'] = ['size']\n# aggs['merchants_state_id_nunique'] = ['size']\n\n# merchant_category_id\n# Low feature importance; might need to remove\n# aggs['merchants_merchant_category_id_std'] = ['min', 'max', 'std']\n\n# numerical_1\n# KS discard features 'numerical_1_mean_max'\naggs['merchants_numerical_1_mean'] = ['min']\n# KS discard features 'numerical_1_sum_max', 'numerical_1_sum_mean'\naggs['merchants_numerical_1_sum'] = ['min']\n\n# numerical_2\n# KS discard features 'numerical_2_mean_max'\naggs['merchants_numerical_2_mean'] = ['min']\n# KS discard features 'numerical_2_sum_mean'\n# aggs['merchants_numerical_2_sum'] = ['mean'] # 'min', 'max', \n\n# 'avg_sales_lag3'\n# KS discard features 'avg_sales_lag3_sum_mean'\naggs['merchants_avg_sales_lag3_sum'] = ['min', 'max'] # 'mean'\naggs['merchants_avg_sales_lag3_mean'] = ['min', 'max']\naggs['merchants_avg_sales_lag3_min'] = ['min', 'max']\n# aggs['merchants_avg_sales_lag3_size'] = ['min', 'max']\n# aggs['merchants_avg_sales_lag3_max'] = ['min', 'max']\n\n# 'avg_purchases_lag3'\n# KS discard features 'avg_purchases_lag3_sum_max', 'avg_purchases_lag3_sum_mean'\naggs['merchants_avg_purchases_lag3_sum'] = ['min'] # 'max', 'mean'\n# KS discard features 'avg_purchases_lag3_mean_max'\naggs['merchants_avg_purchases_lag3_mean'] = ['min'] # 'max'\n# KS discard features 'avg_purchases_lag3_min_min', 'avg_purchases_lag3_min_max'\n# aggs['merchants_avg_purchases_lag3_min'] = ['min', 'max']\n# KS discard features'avg_purchases_lag3_max_max'\naggs['merchants_avg_purchases_lag3_max'] = ['min'] # 'max'\n# aggs['merchants_avg_purchases_lag3_size'] = ['min', 'max']\n\n## 'active_months_lag3'\n# aggs['merchants_active_months_lag3_sum'] = ['mean'] # 'min', 'max',\n# aggs['merchants_active_months_lag3_mean'] = ['mean', 'median'] # 'sum', 'min', 'max'\n# aggs['merchants_active_months_lag3_size'] = ['min', 'max']\n# aggs['merchants_active_months_lag3_min'] = ['min', 'max']\n# aggs['merchants_active_months_lag3_max'] = ['min', 'max']\n\n# 'avg_sales_lag6'\n# KS discard features 'avg_sales_lag6_sum_mean'\naggs['merchants_avg_sales_lag6_sum'] = ['min', 'max'] # 'mean'\naggs['merchants_avg_sales_lag6_mean'] = ['min', 'max']\naggs['merchants_avg_sales_lag6_min'] = ['min', 'max']\n# aggs['merchants_avg_sales_lag6_size'] = ['min', 'max']\n# aggs['merchants_avg_sales_lag6_max'] = ['min', 'max']\n\n# 'avg_purchases_lag6'\n# KS discard features 'avg_purchases_lag6_sum_min', 'avg_purchases_lag6_sum_max', 'avg_purchases_lag6_sum_mean'\n# aggs['merchants_avg_purchases_lag6_sum'] = ['min', 'max', 'mean']\n# KS discard features 'avg_purchases_lag6_mean_min', 'avg_purchases_lag6_max_max'\n# aggs['merchants_avg_purchases_lag6_mean'] = ['min'] # 'max'\n# aggs['merchants_avg_purchases_lag6_max'] = ['max'] # 'min'\naggs['merchants_avg_purchases_lag6_min'] = ['min'] # 'max'\n# aggs['merchants_avg_purchases_lag6_size'] = ['min', 'max']\n\n# 'active_months_lag6'\n# aggs['merchants_active_months_lag6_sum'] = ['mean', 'min', 'max']\n# aggs['merchants_active_months_lag6_mean'] = ['sum', 'min', 'max']\n# aggs['merchants_active_months_lag6_size'] = ['min', 'max']\n# aggs['merchants_active_months_lag6_min'] = ['min', 'max']\n# aggs['merchants_active_months_lag6_max'] = ['min', 'max']\n\n# 'avg_sales_lag12'\n# KS discard features 'avg_sales_lag12_sum_sum', 'avg_sales_lag12_sum_max',\naggs['merchants_avg_sales_lag12_sum'] = ['min', 'mean'] # 'sum', 'max'\naggs['merchants_avg_sales_lag12_mean'] = ['min', 'max']\n# aggs['merchants_avg_sales_lag12_size'] = ['min', 'max']\n# aggs['merchants_avg_sales_lag12_min'] = ['min', 'max']\n# aggs['merchants_avg_sales_lag12_max'] = ['min', 'max']\n\n# 'avg_purchases_lag12'\n# KS discard features 'avg_purchases_lag12_sum_max', 'avg_purchases_lag12_sum_mean'\n# aggs['merchants_avg_purchases_lag12_sum'] = ['max', 'mean'] # 'min'\n# 'avg_purchases_lag12_mean_min', 'avg_purchases_lag12_mean_max'\n# aggs['merchants_avg_purchases_lag12_mean'] = ['min', 'max']\n# KS discard features 'avg_purchases_lag12_min_min', 'avg_purchases_lag12_min_max'\n# aggs['merchants_avg_purchases_lag12_min'] = ['min', 'max']\n# aggs['merchants_avg_purchases_lag12_size'] = ['min', 'max'] \n# aggs['merchants_avg_purchases_lag12_max'] = ['min', 'max']\n\n\n# 'active_months_lag12'\naggs['merchants_active_months_lag12_mean'] = ['sum'] # 'min', 'max'\n# aggs['merchants_active_months_lag12_sum'] = ['mean', 'min', 'max']\n# aggs['merchants_active_months_lag12_size'] = ['min', 'max']\n# aggs['merchants_active_months_lag12_min'] = ['min', 'max']\n# aggs['merchants_active_months_lag12_max'] = ['min', 'max']\n\n# added more features with median for 'installments'\n# aggs['month'] = [ 'min', 'max', 'mean', 'var']\n\n# 'category_1_mean_mean' not useful\nfor col in ['category_2','category_3']: # 'category_1'\n # Replace newmerchant_data with newmerchant_with_merchants_data\n newmerchant_with_merchants_data[col+'_purchase_amount_mean'] = newmerchant_with_merchants_data.groupby([col])['purchase_amount'].transform('mean')\n aggs[col+'_purchase_amount_mean'] = ['mean']\n\n# Explore purchase_amount_sum_mean in 'category_1' which helps to improve on ranking\nfor col in ['category_1']:\n newmerchant_with_merchants_data[col+'_purchase_amount_sum'] = newmerchant_with_merchants_data.groupby([col])['purchase_amount'].transform('sum')\n aggs[col+'_purchase_amount_sum'] = ['mean'] \n\nnew_columns = create_new_columns('new_merchant',aggs)\nprint(\"New aggregated columns :\\n\", new_columns)\ndf_new_merchant_trans_group = newmerchant_with_merchants_data.groupby('card_id').agg(aggs) # newmerchant_data\n\ndf_new_merchant_trans_group.columns = new_columns\ndf_new_merchant_trans_group.reset_index(drop=False,inplace=True)\ndf_new_merchant_trans_group['new_merchant_purchase_date_diff'] = (df_new_merchant_trans_group['new_merchant_purchase_date_max'] - df_new_merchant_trans_group['new_merchant_purchase_date_min']).dt.days\ndf_new_merchant_trans_group['new_merchant_purchase_date_average'] = df_new_merchant_trans_group['new_merchant_purchase_date_diff'] / df_new_merchant_trans_group['new_merchant_card_id_size']\n\n# Replace datetime.datetime.today() with df_new_merchant_trans_group['new_merchant_purchase_date_max'] in attempt to improve relevancy\ndf_new_merchant_trans_group['new_merchant_purchase_date_uptonow'] = (datetime.datetime.today() - \\\n df_new_merchant_trans_group['new_merchant_purchase_date_max']).dt.days\n\n# Explore new features for rank ascend\ndf_new_merchant_trans_group['new_card_last_woy'] = df_new_merchant_trans_group['new_merchant_purchase_date_max'].dt.weekofyear\ndf_new_merchant_trans_group['new_card_last_doy'] = df_new_merchant_trans_group['new_merchant_purchase_date_max'].dt.dayofyear\ndf_new_merchant_trans_group['new_card_last_day'] = df_new_merchant_trans_group['new_merchant_purchase_date_max'].dt.day\n\n# --\n# Reference : https://www.kaggle.com/raddar/towards-de-anonymizing-the-data-some-insights\ndf_new_merchant_trans_group['new_merchant_purchase_amount_new'] = np.round((newmerchant_data['purchase_amount'] / 0.00150265118) + 497.06, 8)\n\n# Check new feature effects on RMSE\ntarget = 'new_merchant_purchase_amount_new'\ndf_new_merchant_trans_group['new_merchant_card_sum_purchase_amount_new'] = df_new_merchant_trans_group['card_id'].map(df_new_merchant_trans_group.groupby(['card_id'])[target].sum())\n\ndf_new_merchant_trans_group['new_merchant_card_mean_purchase_amount_new'] = df_new_merchant_trans_group['card_id'].map(df_new_merchant_trans_group.groupby(['card_id'])[target].mean())\ndf_new_merchant_trans_group['new_merchant_card_median_purchase_amount_new'] = df_new_merchant_trans_group['card_id'].map(df_new_merchant_trans_group.groupby(['card_id'])[target].median())\ndf_new_merchant_trans_group['new_merchant_card_min_purchase_amount_new'] = df_new_merchant_trans_group['card_id'].map(df_new_merchant_trans_group.groupby(['card_id'])[target].min())\ndf_new_merchant_trans_group['new_merchant_card_max_purchase_amount_new'] = df_new_merchant_trans_group['card_id'].map(df_new_merchant_trans_group.groupby(['card_id'])[target].max())\ndf_new_merchant_trans_group['new_merchant_card_std_purchase_amount_new'] = df_new_merchant_trans_group['card_id'].map(df_new_merchant_trans_group.groupby(['card_id'])[target].std())\n\n# KS discard feature \n# 'new_merchant_card_sum_month_lag_1_purchase_amount_new', 'new_merchant_card_sum_month_lag_2_purchase_amount_new'\n# df_new_merchant_trans_group['month_lag'] = newmerchant_data['month_lag']\n# df_new_merchant_trans_group['new_merchant_card_sum_month_lag_1_purchase_amount_new'] =\\\n# df_new_merchant_trans_group['card_id'].map(df_new_merchant_trans_group[df_new_merchant_trans_group['month_lag']==1].groupby(['card_id'])[target].sum())\n# df_new_merchant_trans_group['new_merchant_card_sum_month_lag_2_purchase_amount_new'] =\\\n# df_new_merchant_trans_group['card_id'].map(df_new_merchant_trans_group[df_new_merchant_trans_group['month_lag']==2].groupby(['card_id'])[target].sum())\n\n# Explore new building feature, 'card_purchase_date_max_ratio', for rank ascend\ndf_new_merchant_trans_group['new_purchase_date_int'] = newmerchant_with_merchants_data['purchase_date'].astype(np.int64) * 1e-9\ndf_new_merchant_trans_group['new_card_purchase_date_max'] = \\\n df_new_merchant_trans_group.groupby(['card_id'])['new_purchase_date_int'].transform('max')\n# --\n\nprint()\nprint(\"Shape of df_new_merchant_trans_group :\\n\", df_new_merchant_trans_group.shape)\nprint(\"df_new_merchant_trans_group columns :\\n\", df_new_merchant_trans_group.columns)\n\n# Remove aggs['merchants_merchant_category_id_std'] = ['min', 'max', 'std'] in v54","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create more aggregated features from 'new_merchant_purchase_amount_new","metadata":{}},{"cell_type":"code","source":"# df_new_merchant_trans_group['new_merchant_purchase_date_diff']\ncheck_missing_values(df_new_merchant_trans_group)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get ready for training models\nX_train = df_train.merge(df_new_merchant_trans_group,on='card_id',how='left')\nX_test = df_test.merge(df_new_merchant_trans_group,on='card_id',how='left')\n\nX_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Release unnecessary data to reduce memory used\ndel df_new_merchant_trans_group, merchants_data, df_train, df_test\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train['hist_purchase_date_max'].head(2), X_test['hist_purchase_date_max'].head(2)\n\n# X_train['new_merchant_purchase_date_max'].head(2), X_test['new_merchant_purchase_date_max'].head(2)\n\n# X_train['hist_purchase_amount_max'].head(2), X_test['hist_purchase_amount_max'].head(2)\n\n# X_train['new_merchant_purchase_amount_max'].head(2), X_test['new_merchant_purchase_amount_max'].head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train = df_train.merge(df_merchant_category, on='merchant_category_id', how='left')\n# df_test = df_test.merge(df_merchant_category, on='merchant_category_id', how='left')\n\n# df_train.shape, df_test.shape\n# check_missing_values(X_train), check_missing_values(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Concatenate the categorical features\n# Numerical representations of the nominal feature_1, feature_2, feature_3\n# X_train = pd.concat([X_train, ohe_train_df_1, ohe_train_df_2, ohe_train_df_3], axis=1, sort=False)\n# X_test = pd.concat([X_test, ohe_test_df_1, ohe_test_df_2, ohe_test_df_3], axis=1, sort=False)\n\n# del ohe_train_df_1, ohe_train_df_2, ohe_train_df_3\n# del ohe_test_df_1, ohe_test_df_2, ohe_test_df_3\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train[['hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max']]\n# X_train[['hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max']].isna().any()\n# X_test[['hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max']].isna().any()\n\n# print(\"Missing purchase date (min-max) in X_train : \")\n# print(\"min mode : {}\\nmax mode : {}\".format(X_train['new_merchant_purchase_date_min'].mode(),\n# X_train['new_merchant_purchase_date_max'].mode()))\n# print(\"-\" * 80)\n# print(\"Missing purchase date (min-max) in X_test : \")\n# print(\"min mode : {}\\nmax mode : {}\".format(X_test['new_merchant_purchase_date_min'].mode(),\n# X_test['new_merchant_purchase_date_max'].mode()))\n\n# type(X_train['new_merchant_purchase_date_min'].mode()[0]), type(X_train['new_merchant_purchase_date_max'].mode()[0])\n\n# print(\"Purchase data in historical_data\")\n# print(historical_data['purchase_date'].min(), historical_data['purchase_date'].max())\n# print(\"\\nPurchase data in newmerchant_data\")\n# print(newmerchant_data['purchase_date'].min(), newmerchant_data['purchase_date'].max())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Impute missing purchase min-max values in train and test data","metadata":{}},{"cell_type":"code","source":"# X_train[['hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max',\n# 'hist_purchase_amount_min', 'hist_purchase_amount_max', 'new_merchant_purchase_amount_min', 'new_merchant_purchase_amount_max']].info()\n\n# X_test[['hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max',\n# 'hist_purchase_amount_min', 'hist_purchase_amount_max', 'new_merchant_purchase_amount_min', 'new_merchant_purchase_amount_max']].info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train['hist_purchase_date_min'].fillna(historical_data['purchase_date'].min(), inplace = True)\nX_train['hist_purchase_date_max'].fillna(historical_data['purchase_date'].max(), inplace = True)\nX_train['new_merchant_purchase_date_min'].fillna(newmerchant_data['purchase_date'].min(), inplace = True)\nX_train['new_merchant_purchase_date_max'].fillna(newmerchant_data['purchase_date'].max(), inplace = True)\n\nX_train['hist_purchase_amount_min'].fillna(historical_data['purchase_amount'].min(), inplace = True)\nX_train['hist_purchase_amount_max'].fillna(historical_data['purchase_amount'].max(), inplace = True)\nX_train['new_merchant_purchase_amount_min'].fillna(newmerchant_data['purchase_amount'].min(), inplace = True)\nX_train['new_merchant_purchase_amount_max'].fillna(newmerchant_data['purchase_amount'].max(), inplace = True)\n\nX_test['hist_purchase_date_min'].fillna(historical_data['purchase_date'].min(), inplace = True)\nX_test['hist_purchase_date_max'].fillna(historical_data['purchase_date'].max(), inplace = True)\nX_test['new_merchant_purchase_date_min'].fillna(newmerchant_data['purchase_date'].min(), inplace = True)\nX_test['new_merchant_purchase_date_max'].fillna(newmerchant_data['purchase_date'].max(), inplace = True)\n\nX_test['hist_purchase_amount_min'].fillna(historical_data['purchase_amount'].min(), inplace = True)\nX_test['hist_purchase_amount_max'].fillna(historical_data['purchase_amount'].max(), inplace = True)\nX_test['new_merchant_purchase_amount_min'].fillna(newmerchant_data['purchase_amount'].min(), inplace = True)\nX_test['new_merchant_purchase_amount_max'].fillna(newmerchant_data['purchase_amount'].max(), inplace = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # 'hist_card_sum_month_lag_0_purchase_amount_new'\n# target = 'hist_sum_purchase_amount_new'\n# X_train['month_lag'] = historical_data['month_lag']\n# X_train['hist_card_sum_month_lag_0_purchase_amount_new'] =\\\n# X_train['card_id'].map(X_train[X_train['month_lag']==0].groupby(['card_id'])[target].sum())\n# TBD","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Releases all memories in preparation for model training\ndel historical_data, newmerchant_data # df_new_merchant_trans_group, merchants_data, df_train, df_test, train_data, test_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[['hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max',\n 'hist_purchase_amount_min', 'hist_purchase_amount_max', 'new_merchant_purchase_amount_min', 'new_merchant_purchase_amount_max']].info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test[['hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max',\n 'hist_purchase_amount_min', 'hist_purchase_amount_max', 'new_merchant_purchase_amount_min', 'new_merchant_purchase_amount_max']].info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The ratios does not improve the overall RMSE\n# purchase_date in timestamp format\n# for df in [X_train, X_test]:\n# df['hist_purchase_date_min_month'] = df['hist_purchase_date_min'].dt.month\n# df['hist_purchase_date_max_month'] = df['hist_purchase_date_max'].dt.month\n# df['new_merchant_purchase_date_min_month'] = df['new_merchant_purchase_date_min'].dt.month\n# df['new_merchant_purchase_date_max_month'] = df['new_merchant_purchase_date_max'].dt.month\n \n# df['hist_purchase_date_maxmin_month_ratio'] = df['hist_purchase_date_max_month'] / df['hist_purchase_date_min_month']\n# df['new_merchant_purchase_date_maxmin_month_ratio'] = df['new_merchant_purchase_date_max_month'] / \\\n# df['new_merchant_purchase_date_min_month']\n\n# X_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## More Feature Engineering with 'first_active_month'","metadata":{}},{"cell_type":"code","source":"for df in [X_train, X_test]:\n df_first_active_month_max = pd.to_datetime(df['first_active_month'].max())\n \n df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n df['first_active_month_dayofweek'] = df['first_active_month'].dt.dayofweek\n df['first_active_month_weekofyear'] = df['first_active_month'].dt.weekofyear\n df['first_active_month_month'] = df['first_active_month'].dt.month\n \n # df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n # KS discard feature 'elapsed_time' = ((df_first_active_month_max - df['first_active_month']).dt.days)//30\n df['first_active_month_today_elapsed_time'] = ((datetime.datetime.today() - df['first_active_month']).dt.days)//30\n \n # Explore interactions with 'hist_purchase_date_min', 'new_merchant_purchase_date_min' and 'first_active_month'\n df['hist_pdmin_active_month_diff'] = ((df['hist_purchase_date_min'] - df['first_active_month']).dt.days)//30\n df['new_merchant_pdmin_active_month_diff'] = ((df['new_merchant_purchase_date_min'] - df['first_active_month']).dt.days)//30\n\nX_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert Timestamp to int64 with imputed missing values\nfor f in ['hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max']:\n print(type(X_train[f][0]))\n print(type(X_test[f][0]))\n X_train[f] = X_train[f].astype(np.int64) * 1e-9\n X_test[f] = X_test[f].astype(np.int64) * 1e-9","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train[['hist_card_id_size', 'new_merchant_card_id_size']].head(2)\n# X_train[['new_merchant_card_id_size', 'new_merchant_purchase_date_max']].head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* # Feature Interactions (ratios and differences)","metadata":{}},{"cell_type":"code","source":"# List all columns in dataframe\n# list(X_train.columns.values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This strong 'new_hist_purchase_date_max_ratio' feature elevates the ranking by 10%+\n# Added 10 new KS features\nfor df in [X_train, X_test]:\n # purchase_date\n df['new_hist_purchase_date_max_ratio'] = df['new_merchant_purchase_date_max'] / df['hist_purchase_date_max']\n df['new_hist_purchase_date_min_ratio'] = df['new_merchant_purchase_date_min'] / df['hist_purchase_date_min']\n df['hist_purchase_date_maxmin_ratio'] = df['hist_purchase_date_max'] / df['hist_purchase_date_min']\n df['new_purchase_date_maxmin_ratio'] = df['new_merchant_purchase_date_max'] / df['new_merchant_purchase_date_min']\n \n # purchase_amount\n df['new_hist_purchase_amount_max_diff'] = df['new_merchant_purchase_amount_max'] - df['hist_purchase_amount_max']\n df['new_hist_purchase_amount_min_diff'] = df['new_merchant_purchase_amount_min'] - df['hist_purchase_amount_min']\n df['new_hist_purchase_amount_max_ratio'] = df['new_merchant_purchase_amount_max'] / df['hist_purchase_amount_max']\n df['new_hist_purchase_amount_min_ratio'] = df['new_merchant_purchase_amount_min'] / df['hist_purchase_amount_min']\n \n df['new_pa_max_hist_pa_sum_ratio'] = df['new_merchant_purchase_amount_max'] / df['hist_purchase_amount_sum']\n df['new_hist_card_id_total'] = df['new_merchant_card_id_size'] + df['hist_card_id_size']\n\n # Need to produce 'card_purchase_date_max_ratio'\n\nX_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Did not improve on the RMSE ranking\nfor df in [X_train, X_test]:\n df['new_hist_purchase_date_uptonow_ratio'] = df['new_merchant_purchase_date_uptonow'] / df['hist_purchase_date_uptonow']\n df['new_hist_purchase_date_uptonow_diff'] = df['new_merchant_purchase_date_uptonow'] - df['hist_purchase_date_uptonow']\n \nX_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train[['new_card_purchase_date_max', 'hist_card_purchase_date_max']].head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Did not improve on the RMSE ranking\nX_train['card_purchase_date_max_ratio'] = X_train['new_card_purchase_date_max'] + X_train['hist_card_purchase_date_max']\nX_test['card_purchase_date_max_ratio'] = X_test['new_card_purchase_date_max'] + X_test['hist_card_purchase_date_max']\n\nX_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# features_correlation = pd.DataFrame(historical_data, columns=['category_1','city_id',\n# 'category_2','state_id',\n# 'category_3','installments'])\n\n# colormap = plt.cm.RdBu\n# plt.figure(figsize=(10,8))\n# plt.title('Pearson Correlation of anonymise features', y=1.05, size=15)\n# sns.heatmap(features_correlation.astype(float).corr(),linewidths=0.1,vmax=1.0, \n# square=True, cmap=colormap, linecolor='white', annot=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Did not improve on the RMSE ranking\n# for f in ['new_merchant_purchase_date_max',\n# 'hist_purchase_date_max',]:\n# X_train[f +'_int'] = X_train[f].astype(np.int64) * 1e-9\n# X_test[f +'_int'] = X_test[f].astype(np.int64) * 1e-9\n \n# X_train['card_purchase_date_max_ratio'] = X_train['new_merchant_purchase_date_max_int'] / X_train['hist_purchase_date_max_int']\n# X_test['card_purchase_date_max_ratio'] = X_test['new_merchant_purchase_date_max_int'] / X_test['hist_purchase_date_max_int']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train['new_merchant_merchants_category_1_min_min'].mode()[0]\n\n# # Fill missing values for min-max purchase date newmerchant_data from after merging with newmmerchant_data for the card_ids in train_data\n# X_train['new_merchant_merchants_category_1_min_min'].fillna(newmerchant_data['category_1'].min(), inplace = True)\n# X_train['new_merchant_merchants_category_1_min_max'].fillna(newmerchant_data['category_1'].min(), inplace = True)\n# X_train['new_merchant_merchants_category_1_max_min'].fillna(newmerchant_data['category_1'].max(), inplace = True)\n# X_train['new_merchant_merchants_category_1_max_max'].fillna(newmerchant_data['category_1'].max(), inplace = True)\n\n# X_test['new_merchant_merchants_category_1_min_min'].fillna(newmerchant_data['category_1'].min(), inplace = True)\n# X_test['new_merchant_merchants_category_1_min_max'].fillna(newmerchant_data['category_1'].min(), inplace = True)\n# X_test['new_merchant_merchants_category_1_max_min'].fillna(newmerchant_data['category_1'].max(), inplace = True)\n# X_test['new_merchant_merchants_category_1_max_max'].fillna(newmerchant_data['category_1'].max(), inplace = True)\n\n# X_train[['new_merchant_merchants_category_1_min_min', 'new_merchant_merchants_category_1_min_max',\n# 'new_merchant_merchants_category_1_max_min', 'new_merchant_merchants_category_1_max_max']].info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# excluded_features = ['first_active_month', 'card_id', 'target', 'date', 'year']\n# Excluse non numeric features\n# excluded_features = ['first_active_month', 'card_id', 'target', 'hist_purchase_date_min', 'hist_purchase_date_max']\n# train_features = [c for c in df_train.columns if c not in excluded_features]\n\n# 'hist_purchase_date_min', 'hist_purchase_date_max', 'new_merchant_purchase_date_min', 'new_merchant_purchase_date_max'\n# Consider removal of these 'new_merchant_purchase_date_max', 'hist_purchase_date_max' after getting their ratio\nexcluded_features = ['first_active_month', 'card_id', 'target', 'outliers']\ntrain_features = [c for c in X_train.columns if c not in excluded_features]\n\nprint(\"{} features used for training :\\n{}\".format(len(train_features), train_features))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filled mean information for the missing cards from newmerchant_data\n# Fill missing values with mean values; maybe use median value\nfor col in train_features:\n for df in [X_train, X_test]:\n if df[col].dtype == \"float64\":\n df[col] = df[col].fillna(df[col].median()) # mean","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_missing_values(X_train), check_missing_values(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Need to remove 'hist_card_std_purchase_amount_new', 'new_merchant_card_std_purchase_amount_new'\nX_train.drop(columns=['hist_card_std_purchase_amount_new', 'new_merchant_card_std_purchase_amount_new'], axis=1, inplace=True)\nX_test.drop(columns=['hist_card_std_purchase_amount_new', 'new_merchant_card_std_purchase_amount_new'], axis=1, inplace=True)\nX_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* # Kolmogorov-Smirnov\n> - It computes on 2 samples to test for the null hypothesis that 2 independent samples are drawn from the same continuous distribution.
\n> - The statistic test computes the KS statistic and pvalue. If the p value is less than .1 the null hypothesis is rejected.
\n> - Reducing from 190 to 126 features gives a better scoring (from 3.69219 to 3.69101) with p value is less than 0.1.
","metadata":{}},{"cell_type":"code","source":"%%time\nfrom tqdm import tqdm\n# Kolmogorov-Smirnov test for goodness of fit\n# In this case we are sampling values from corresponding columns in train and test data frame followed by storing their pvalues in Se.\n# If the p value is less than 0.05 (0.1) the null hypothesis is rejected.\n# Columns with pvalues less than 0.05 (0.1) are added to the list_discarded.\nlist_p_value =[]\n\nfor i in tqdm(X_train[train_features].columns):\n list_p_value.append(ks_2samp(X_train[i] , X_test[i])[1])\n\nSe = pd.Series(list_p_value, index = X_train[train_features].columns).sort_values() \nlist_discarded = list(Se[Se < 0.1].index)\n\n# 'hist_category_3_purchase_amount_sum_mean' and 'hist_purchase_amount_max'\nlist_discarded","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list_discarded)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- #### Code snippets and data observations ","metadata":{}},{"cell_type":"code","source":"# There are card_id in train_data but not in newmerchant_data\n# newmerchant_data.loc[newmerchant_data['card_id'] == missing_sum_card_id]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are duplicate card_ids in historical_data with 29,112,361 transactions, unique values are 325,540\n# There are missing values in category_3', 'merchant_id' and 'category_2'\n# Replace missing values with mode values for category_3', 'merchant_id' and 'category_2'\n# When inplace = True, the data is modified in place,\n# which means it will return nothing and the dataframe is now updated.","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Need to impute values which is the challenge for prediction\n\n# Stable documentation : https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.merge.html#pandas.merge\n# Merge train_data[['card_id','target']] with historical_data to give new historical_data\n# merging target value of card_id for each transaction in historical_transactions Data; using union of keys from both frames\n# historical_data = pd.merge(historical_data, train_data[['card_id','target']], how = 'outer', on = 'card_id')\n\n# # merging target value of card_id for each transaction in new_merchants_transactions Data\n# newmerchant_data = pd.merge(newmerchant_data, train_data[['card_id','target']], how = 'outer', on = 'card_id')\n\n# historical_data['target'].isnull().sum() # / historical_data.shape[0] = 38.07%\n# historical_data['target'].describe()\n\n# for row in range(historical_data.shape[0]):\n# if (historical_data['card_id'][row] == test_data['card_id'][10]):\n# print(row)\n\n# for card_id[0], 'C_ID_0ab67a22ab' : there are 1304310 - 1304243 = 67 historical transactions\n# for card_id[10], 'C_ID_4859ac9ed5' : there are 23622204 - 23622180 = 24 historical transactions\n# historical_data.loc[[1304243]]\n\n# reset_index() to keep the 'card_id' column that appears with X.columns\n# X = partial_historical_data_target.groupby('card_id').mean().reset_index()\n\n# groupby_card_id.isnull().sum()\n\n# Merge test_data[['card_id']] with partial_historical_data without target to give new test_data_intersect\n# test_data_intersect = pd.merge(historical_data, test_data[['card_id']], how = 'inner', on = 'card_id')\n# test_data_intersect\n\n# Drop the non-numeric features\n# partial_historical_data = historical_data_target.drop(columns=['target', 'authorized_flag', 'category_1', 'category_3', 'merchant_id', 'purchase_date'])\n\n# historical_data['target'] = historical_data['target'].replace(0, historical_data['target'].mode())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_missing_values(X_train), check_missing_values(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 2 : Attempt novelty : Exploring outliers, Blending and Ensemble Learning (Stacking)","metadata":{}},{"cell_type":"markdown","source":"- #### Load best features, predictions and saved models for stacking and blending","metadata":{}},{"cell_type":"code","source":"# Load all libraries\n# Load best features from lgbm and run in GPU for DL\nX_train = pd.read_csv('../input/test-predictions/X_train_137_KSfeats.csv')\nX_test = pd.read_csv('../input/test-predictions/X_test_136_KSfeats.csv')\nlgbm_pred_df = pd.read_csv('../input/test-predictions/lgbm_opt2_130KSfeats_K5.csv')\n\nX_train.shape, X_test.shape, lgbm_pred_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:30:26.330022Z","iopub.execute_input":"2021-11-09T10:30:26.330374Z","iopub.status.idle":"2021-11-09T10:30:38.257442Z","shell.execute_reply.started":"2021-11-09T10:30:26.330346Z","shell.execute_reply":"2021-11-09T10:30:38.256303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"excluded_features = ['first_active_month', 'card_id', 'target', 'outliers']\ntrain_features = [c for c in X_train.columns if c not in excluded_features]\n\nprint(\"{} Features used for training : \\n{}\".format(len(train_features), train_features))","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:30:38.259232Z","iopub.execute_input":"2021-11-09T10:30:38.259570Z","iopub.status.idle":"2021-11-09T10:30:38.266089Z","shell.execute_reply.started":"2021-11-09T10:30:38.259537Z","shell.execute_reply":"2021-11-09T10:30:38.265042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove features from loaded best 137KS features in attempt to get better CV RMSE scores\nlist_discarded = ['hist_purchase_amount_max', 'hist_w2v_hist_merchant_id_1_std',\n 'hist_category_3_purchase_amount_sum_mean', 'hist_installments_mean']\n\n# ['hist_purchase_amount_max', 'hist_category_3_purchase_amount_sum_mean', 'hist_installments_mean']\nprint(\"No. of train_features :\", len(train_features))\n\nfeatures_to_remove = list_discarded\n# features_to_remove = ['new_merchant_purchase_amount_sum', 'new_merchant_merchants_most_recent_purchases_range_min_max',\n# 'new_merchant_merchants_avg_purchases_lag3_sum_min', 'new_merchant_merchants_avg_sales_lag6_mean_max',\n# 'new_merchant_merchants_most_recent_sales_range_max_min', 'new_merchant_w2v_new_state_id_0_std',\n# 'new_merchant_card_id_size', 'new_merchant_w2v_new_state_id_1_std',\n# 'new_merchant_merchants_avg_sales_lag3_mean_max', 'new_merchant_merchants_avg_purchases_lag6_sum_max']\n\ntrain_features = list(set(train_features) - set(features_to_remove))\n\nprint(\"No. of train_features after removing some low important features :\", len(train_features))\n\nlen(set(features_to_remove)), len(set(train_features))","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:30:38.267682Z","iopub.execute_input":"2021-11-09T10:30:38.268033Z","iopub.status.idle":"2021-11-09T10:30:38.284283Z","shell.execute_reply.started":"2021-11-09T10:30:38.267999Z","shell.execute_reply":"2021-11-09T10:30:38.283164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data for training and prediction\nX = X_train.copy()\ny = X['target'] # pandas.core.series.Series\ncard_ids = X_test[\"card_id\"].copy()\n\nX.shape, X_train.shape, X[train_features].shape, X_test[train_features].shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:30:47.841114Z","iopub.execute_input":"2021-11-09T10:30:47.841508Z","iopub.status.idle":"2021-11-09T10:30:48.073043Z","shell.execute_reply.started":"2021-11-09T10:30:47.841473Z","shell.execute_reply":"2021-11-09T10:30:48.072039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploring with outliers in training data\n- #### Observations :\n> - The target value is almost normally distributed with bunch of outlier value near -30.\n> - This distribution indicates that the target value is normalized with pre-decided mean and standard deviation.\n> - 2,207 rows with target values less than -33 (1.09%) in \"train.csv\"\n> - Values range from '-33.2' to '17.9'\n> - '-33.219281' seems like an outlier point\n> - Other values less than '-10' also seem like outliers due to very less in number\n> - All values above '10' might also be outliers","metadata":{}},{"cell_type":"code","source":"loyality_score = X['target']\nax = loyality_score.plot.hist(bins=20, figsize=(6, 5))\n_ = ax.set_title(\"target histogram\")\nplt.show()\n\nfig, axs = plt.subplots(1,2, figsize=(12, 5))\n_ = loyality_score[loyality_score > 10].plot.hist(ax=axs[0])\n_ = axs[0].set_title(\"target histogram for values greater than 10\")\n_ = loyality_score[loyality_score < -10].plot.hist(ax=axs[1])\n_ = axs[1].set_title(\"target histogram for values less than -10\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (A) Method 1 : LGBM with non-outliers\n> - training data instances with ‘target’ < -33 are marked as outliers (2,207 of them)\n> - Predictions made with test set\n> - Perform the average between predictions from LGBM with non-outliers and best single model, LGBM, with complete training data
\n(Average blending 'target' predictions from LGBM trained with non-outliers and complete data set separately)","metadata":{}},{"cell_type":"code","source":"# There are 2207 rows with target values less than -30 (1.09%)\nX.loc[X['target'] < -33, 'target'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X['outliers'] = 0\n\n# Consider the target values less than -30 are outliers\nX.loc[X['target'] < -33, 'outliers'] = 1\nX['outliers'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting the rows that does not have null values for 'target'\noutlier_train_df = X[X['target'].notnull()]\n\n# All rows in the training set\noutlier_train_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Added a null column 'target' to X_test\nX_test['target'] = np.nan\nX_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting the rows that has null values for 'target'\noutlier_test_df = X_test[X_test['target'].isnull()]\n\nprint(outlier_train_df.shape, outlier_test_df.shape, X.shape, X_test.shape)\n\n# All rows in the test set\noutlier_test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting only the 199,710 rows (non-assigned outlier with 'outliers' column == 0)\nX_nonoutliers = X.loc[X['outliers'] == 0]\ny_nonoutliers = X.loc[X['outliers'] == 0, 'target']\n\nX_nonoutliers.shape, X_nonoutliers[train_features].shape, y_nonoutliers.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train model with only non-outliers to be used for average blending\n- ### Build lgbm regressor with non-outliers training set\n> #### Observations :\n> - #### The LGBM model overfits if we just omit the rows from the training set with 'target' value less than -33.","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n\nnp.random.seed(4590)\n\n# Baselined lgbm params\nlgbm_params = {'num_leaves': 111,\n 'min_data_in_leaf': 150, # 149\n 'objective':'regression',\n 'max_depth': 9,\n 'learning_rate': 0.005,\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.7522,\n \"bagging_freq\": 1,\n \"bagging_fraction\": 0.7083,\n \"bagging_seed\": 11,\n \"metric\": 'rmse',\n \"lambda_l1\": 0.2634,\n \"random_state\": 133,\n \"verbosity\": -1}\n\n# Does not improve rank with 'learning_rate': 0.001 or # 0.002\nlgbm_params_opt4 = {'num_leaves': 111,\n 'min_data_in_leaf': 150,\n 'objective':'regression',\n 'max_depth': 9,\n 'learning_rate': 0.001, # 0.002,\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.7522,\n \"bagging_freq\": 1,\n \"bagging_fraction\": 0.7083,\n \"bagging_seed\": 11,\n \"metric\": 'rmse',\n \"lambda_l1\": 0.2634,\n \"verbosity\": -1,\n # \"nthread\": 4,\n \"random_state\": 4590} \n\n# Optimised lgbm params : improvement from fine tuning (best at this moment)\nlgbm_params_opt1 = {'num_leaves': 127,\n 'min_data_in_leaf': 148,\n 'objective':'regression',\n 'max_depth': 10,\n 'learning_rate': 0.005, # (best at this moment); 0.001 does not improve\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.7202,\n \"bagging_freq\": 1,\n \"bagging_fraction\": 0.8125,\n \"bagging_seed\": 11,\n \"metric\": 'rmse',\n \"lambda_l1\": 0.3468,\n \"random_state\": 4590, # 133,\n \"verbosity\": -1}\n\n# Optimised lgbm params : check if there is improvement\n# Version 41 (177 features) with the following hyperparameter tuning\nlgbm_params_opt2 = {'num_leaves':62,\n 'min_data_in_leaf': 20,\n 'objective':'regression',\n 'max_depth': 8,\n 'learning_rate': 0.001,\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.9,\n \"bagging_freq\": 1,\n \"bagging_fraction\": 0.9,\n \"bagging_seed\": 11,\n \"metric\": 'rmse',\n \"lambda_l1\": 0.1,\n \"verbosity\": -1,\n # \"nthread\": 4,\n \"random_state\": 4590} ","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:33:52.965855Z","iopub.execute_input":"2021-10-28T12:33:52.966218Z","iopub.status.idle":"2021-10-28T12:33:54.558077Z","shell.execute_reply.started":"2021-10-28T12:33:52.966188Z","shell.execute_reply":"2021-10-28T12:33:54.556825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use by multiple methods and stacking variants with different random_state \ndef lgbm_predictions(Xtrain_df, Xtest_df, y, lgbm_params, n_splits):\n categorical_feats = ['feature_1', 'feature_2', 'feature_3']\n num_round = 10000 # 1000\n folds = KFold(n_splits=n_splits, shuffle=True, random_state=4590)\n\n oof = np.zeros(len(Xtrain_df))\n lgbm_pred = np.zeros(len(Xtest_df))\n start = time.time()\n feature_importance_df = pd.DataFrame()\n\n for fold_, (trn_idx, val_idx) in enumerate(folds.split(Xtrain_df.values, y.values)):\n print(\"LGBM fold n°{}\".format(fold_))\n print(\"-\" * 80)\n\n trn_data = lgb.Dataset(Xtrain_df.iloc[trn_idx], # [train_features]\n label=y.iloc[trn_idx],\n categorical_feature=categorical_feats)\n val_data = lgb.Dataset(Xtrain_df.iloc[val_idx], # [train_features]\n label=y.iloc[val_idx],\n categorical_feature=categorical_feats)\n\n lgbm_reg = lgb.train(lgbm_params,\n trn_data,\n num_round,\n valid_sets = [trn_data, val_data],\n verbose_eval=100,\n early_stopping_rounds = 200)\n\n oof[val_idx] = lgbm_reg.predict(Xtrain_df.iloc[val_idx], # [train_features]\n num_iteration=lgbm_reg.best_iteration)\n\n fold_importance_df = pd.DataFrame()\n fold_importance_df[\"feature\"] = Xtrain_df.columns # train_features\n fold_importance_df[\"importance\"] = lgbm_reg.feature_importance()\n fold_importance_df[\"fold\"] = fold_ + 1\n feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n\n lgbm_pred += lgbm_reg.predict(Xtest_df,\n num_iteration=lgbm_reg.best_iteration) / folds.n_splits\n print()\n print(\"CV RMSE score: {:<8.5f}\".format(mean_squared_error(oof, y_nonoutliers)**0.5))\n \n return lgbm_pred","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:33:55.496018Z","iopub.execute_input":"2021-10-28T12:33:55.496387Z","iopub.status.idle":"2021-10-28T12:33:55.509802Z","shell.execute_reply.started":"2021-10-28T12:33:55.496346Z","shell.execute_reply":"2021-10-28T12:33:55.508784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport time\n# Wall time: 14min 17s (using lgbm_params_opt1)\n# Wall time: 1h 4min 11s (using lgbm_params_opt2)\nn_splits = 5\n\n# lgbm_params_opt1 gives best public score\nlgbm_nonoutliers_pred = lgbm_predictions(X_nonoutliers[train_features], X_test[train_features],\n y_nonoutliers, lgbm_params_opt1, n_splits)\n\nlgbm_nonoutliers_pred.shape, X_nonoutliers[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Overfits without the outliers data\n# Early stopping, best iteration is:\n# [1803]\ttraining's rmse: 1.45259\tvalid_1's rmse: 1.52255\n# CV RMSE score: 1.55372 (lgbm opt2) <--\n# CV RMSE score: 1.55693 (lgbm opt2)\n# CV RMSE score: 1.55512 (lgbm opt1)\n# CV RMSE score: 1.55523 (lgbm opt1)\n# CV RMSE score: 1.55490 (lgbm opt1, lr=0.001) does not improve\n\n# Early stopping, best iteration is:\n# [5203]\ttraining's rmse: 1.4621\tvalid_1's rmse: 1.52261\n# CV RMSE score: 1.55486 (lgbm opt4, lr=0.002)\n# Wall time: 37min 2s\n\n# [10000]\ttraining's rmse: 1.46497\tvalid_1's rmse: 1.52237\n# CV RMSE score: 1.55490 (lgbm opt4, lr=0.001)\n# Wall time: 1h 5min 16s\n\n# Predicting the outliers correctly should improve on ranking","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### Predictions made with test set","metadata":{}},{"cell_type":"code","source":"# lgbm_predictions_df = lgbm_pred_df['target'].to_frame(), creates dataframe from a pandas.core.series.Series\nlgbm_predictions_df = pd.DataFrame(lgbm_pred_df['target'])\nlgbm_predictions_df.rename({'target' : 'LGBM_Pred'}, axis=1, inplace=True)\n\nlgbm_nonoutliers_pred_df = pd.DataFrame(lgbm_nonoutliers_pred, columns = ['LGBM_NOutlier_Pred'])\nconcat_test_pred = pd.concat([lgbm_predictions_df, lgbm_nonoutliers_pred_df], axis=1, sort=False)\n\nconcat_test_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### Perform the average between predictions from LGBM with non-outliers and best single model, LGBM, with complete training data
\n#### (Average blending 'target' predictions from LGBM trained with non-outliers and complete data set separately)","metadata":{}},{"cell_type":"code","source":"# Get the mean value of the 2 ML models\nconcat_test_pred['Avg_Pred'] = concat_test_pred.mean(axis=1)\nconcat_test_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[train_features].shape, X_test[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Average predicted values from LGBM with non-outliers (>= -33) in target and complete merged dataset\n# # Producing worst ranking than single LGBM with Public score : 3.74731\ncreate_file_for_submission(\"lgbm_opt_blending_nonoutliers_130KSfeat.csv\", card_ids, concat_test_pred['Avg_Pred'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (B) Method 2 : Build a Classifier to predict outliers\n> - #### LGBM Classifier\n> - #### LGBM predictions that are > 0.5 will be considered as outliers\n> - #### Changing the prediction target values for outliers\n> - #### Blend (1) LGBM predictions with (2) LGBM Classifier predicted non-outlier dataset with adjusted 'target","metadata":{}},{"cell_type":"code","source":"# Training to predict outliers on test set\n# 'outliers' = 1 means the training data instance is an outlier with 'target' < -33\n# y is re-assigned \ny = X['outliers']\ny.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### LGBM Classifier","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\n\n# Basic LGBM classifier\nclf = lgb.LGBMClassifier()\nclf.fit(X[train_features], y)\n\ny_pred = clf.predict(X_test[train_features])\nprint(\"Shape of outlier prediction :\", y_pred.shape)\n\nlgbm_outlier_pred_df = pd.DataFrame(y_pred, columns = ['LGBM_Outlier_Pred'])\nlgbm_outlier_pred_df.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[train_features].shape, X_test[train_features].shape, X_train.shape, X_test.shape, X.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### LGBM predictions that are > 0.5 will be considered as outliers ","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\n# StratifiedKFold for binary or multiclass\nfrom sklearn.model_selection import StratifiedKFold\nnp.random.seed(7878)\n\n# Best results with params\nlgbm_classifier_params = {'num_leaves': 62,\n 'objective':'binary',\n 'metric': 'auc',\n 'max_depth': 8,\n 'learning_rate': 0.01,\n \"boosting_type\": \"gbdt\",\n \"reg_lambda\": 0.2634,\n \"random_state\": 7878, # 133\n \"verbosity\": -1}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom sklearn.model_selection import train_test_split\n\nimport time\ncategorical_feats = ['feature_1', 'feature_2', 'feature_3'] # 'feature_1'\nnum_round = 10000\nn_splits= 5\n\nfolds = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=4590)\n\noof = np.zeros(len(X[train_features]))\nlgbm_outlier_pred = np.zeros(len(X_test[train_features]))\nstart = time.time()\nfeature_importance_df = pd.DataFrame()\n\n# *** This will replace X_train with only 'train_features'; use X instead ***\nX_train, X_valid, y_train, y_valid = train_test_split(X[train_features], y, test_size=0.2, random_state=4590)\ntrain_data = lgb.Dataset(X_train, label=y_train)\nvalid_data = lgb.Dataset(X_valid, label=y_valid, reference=train_data)\n\nparams = {'metric' : lgbm_classifier_params['metric'],\n 'objective' : lgbm_classifier_params['objective'],\n 'verbosity' : lgbm_classifier_params['verbosity'],\n \n # \"reg_lambda\": lgbm_classifier_params['reg_lambda'],\n # Gives the best AUC (lr=0.01) and ranking with public score : 3.68346\n 'learning_rate': lgbm_classifier_params['learning_rate']\n # Gives better AUC 0.90876 but public score : 3.69543\n # 'max_depth' : lgbm_classifier_params['max_depth']\n } \n\nlgbm_outlier_reg = lgb.train(params, # lgbm_classifier_params,\n train_data,\n valid_sets = [valid_data],\n num_boost_round = num_round,\n early_stopping_rounds = 100,\n valid_names=['valid'])\n\nscore = lgbm_outlier_reg.best_score['valid']['auc']\nprint('AUC_score :', score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Early stopping, best iteration is:\n# [692]\tvalid's auc: 0.907677\n# AUC_score : 0.9076771007174149\n# Wall time: 44.8 s\n\n# With max_depth = 8\n# Early stopping, best iteration is:\n# [903]\tvalid's auc: 0.908763\n# AUC_score : 0.9087629103180834\n# Wall time: 52.4 s\n\n# With 'reg_lambda'\n# Early stopping, best iteration is:\n# [751]\tvalid's auc: 0.908012\n# AUC_score : 0.9080121760338308\n# Wall time: 47.1 s","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict outliers in 'target'\nlgbm_outlier_pred = lgbm_outlier_reg.predict(X_test[train_features])\n# >= 0.5 will be assign value 1 \nlgbm_outlier_pred = lgbm_outlier_pred.round(0)\nlgbm_outlier_pred = lgbm_outlier_pred.astype(int)\n\nlgbm_outlier_pred.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_outlier_pred_df = pd.DataFrame(lgbm_outlier_pred, columns = ['LGBM_Outlier_Pred'])\n\n# Lesser predicted outliers than basic classifier\nlgbm_outlier_pred_df.value_counts()\n\n# LGBM_Outlier_Pred without max_depth params\n# 0 123605\n# 1 18","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_outlier_pred_df[\"card_id\"] = card_ids\nlgbm_outlier_pred_df[\"LGBM_Pred\"] = lgbm_pred_df['target'] # pd.Series(lgbm_predictions)\nlgbm_outlier_pred_df.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show only predicted outliers\nlgbm_outlier_pred_df.loc[lgbm_outlier_pred_df['LGBM_Outlier_Pred']==1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concat_test_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concat_test_pred['outliers'] = concat_test_pred['LGBM_NOutlier_Pred'] >= concat_test_pred['LGBM_NOutlier_Pred'].quantile(0.9)\n# concat_test_pred['outliers'].value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using the earlier created dataframe for averaging the prediction with non-outlier and complete train data set \nconcat_test_pred['LGBM_Outlier_Pred'] = lgbm_outlier_pred\nconcat_test_pred.loc[concat_test_pred['LGBM_Outlier_Pred']==1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concat_test_pred['LGBM_Outlier_Pred'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train['final'] = train['bin_predict'](-33.21928)+(1-train['bin_predict'])train['no_outlier']\n# Classifier AUC: 0.914; no outlier Regression CV:1.545\n\n# Train with non-outliers with lgbm_params_opt2 -> lgbm_nonoutliers_pred = concat_test_pred['LGBM_NOutlier_Pred']\n# Case 1 : Public score : 3.72170 and with LGBM_Pred averaging is 3.69543\n# concat_test_pred['Outlier_adj_Pred'] = concat_test_pred['LGBM_Pred'] + \\\n# concat_test_pred['LGBM_Outlier_Pred']*(-33.219281)\n\n# Case 2 : Public score : 3.68696 and with LGBM_Pred averaging is 3.68706\n# concat_test_pred['Outlier_adj_Pred'] = concat_test_pred['LGBM_Pred'] + \\\n# (concat_test_pred['LGBM_Outlier_Pred'] * concat_test_pred['LGBM_NOutlier_Pred'])\n\n# Case 3 : Public score : 3.70081 and with LGBM_Pred averaging is 3.69074\n# concat_test_pred['Outlier_adj_Pred'] = (1 - concat_test_pred['LGBM_Outlier_Pred']) * concat_test_pred['LGBM_Pred'] + \\\n# (concat_test_pred['LGBM_Outlier_Pred'] * -33.219281)\n\n# Case 4 : Public score : 3.69629 and with LGBM_Pred averaging is 3.68995\n# concat_test_pred['Outlier_adj_Pred'] = (1 - concat_test_pred['LGBM_Outlier_Pred']) * concat_test_pred['-33.219281'] + \\\n# (concat_test_pred['LGBM_Outlier_Pred'] * concat_test_pred['LGBM_NOutlier_Pred'])\n\n# Case 5 : Public score : 3.88467 and with LGBM_Pred averaging is 3.74621\n# concat_test_pred['Outlier_adj_Pred'] = (1 - concat_test_pred['LGBM_Outlier_Pred']) * concat_test_pred['LGBM_NOutlier_Pred'] + \\\n# (concat_test_pred['LGBM_Outlier_Pred'] * -33.219281)\n\n# Train with non-outliers with lgbm_params_opt1 with 'LGBM_NOutlier_Pred' instead of '-33.219281'\n# max_depth = 8\n# Case 1 : Public score : 3.68690, Private score : 3.61803 and \n# with LGBM_Pred averaging Public score : 3.68703, Private score : 3.61809\n# With param 'max_depth' (better)\n# ** Case 1 : Public score : 3.68662, Private score : 3.61804 and \n# with LGBM_Pred averaging Public score : 3.68346, Private score : 3.61926 **\nconcat_test_pred['Outlier_adj_Pred'] = concat_test_pred['LGBM_Pred'] + \\\n (concat_test_pred['LGBM_Outlier_Pred'] * concat_test_pred['LGBM_NOutlier_Pred'])\n\nconcat_test_pred.loc[concat_test_pred['LGBM_Outlier_Pred']==1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_file_for_submission(\"lgbm_opt2_130KS_feats_KFold5_outliers_opt4_lr2_reg_lambda_adj1.csv\",\n card_ids, concat_test_pred['Outlier_adj_Pred'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### Changing the prediction target values for outliers\n> #### final prediction result ＝ (single model LGBM result) + (-33.219281 * binary outlier prediction)","metadata":{}},{"cell_type":"code","source":"lgbm_outlier_pred_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Simply assign value to predicted outlier with '-33.219281' and leave predicted values unchanged for non-ouliters\n# lgbm_outlier_pred_df.loc[lgbm_outlier_pred_df['LGBM_Outlier_Pred']==1, 'LGBM_Preds'] = -33.219281\n# lgbm_outlier_pred_df.loc[lgbm_outlier_pred_df['LGBM_Preds'] == -33.219281]\n\n# The following final result is the different from the above operations by regression result\n# final prediction result ＝ (single model LGBM result) + (-33.219281 * binary outlier prediction)\nlgbm_outlier_pred_df['Outlier_adj_Pred'] = lgbm_outlier_pred_df['LGBM_Pred'] + \\\n lgbm_outlier_pred_df['LGBM_Outlier_Pred']*(-33.219281)\n\nprint(lgbm_outlier_pred_df.loc[lgbm_outlier_pred_df['LGBM_Outlier_Pred']==1].shape)\nlgbm_outlier_pred_df.loc[lgbm_outlier_pred_df['LGBM_Outlier_Pred']==1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concat_test_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concat_test_pred['Outlier_adj_Pred'] = lgbm_outlier_pred_df['Outlier_adj_Pred']\nconcat_test_pred.loc[concat_test_pred['LGBM_Outlier_Pred']==1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- #### Blend (1) LGBM predictions with (2) LGBM Classifier predicted non-outlier dataset with adjusted 'target'","metadata":{}},{"cell_type":"code","source":"concat_test_pred['Avg_Pred'] = concat_test_pred[['LGBM_Pred', 'Outlier_adj_Pred']].mean(axis=1)\n\nprint(concat_test_pred.loc[concat_test_pred['LGBM_Outlier_Pred']==1].shape)\nconcat_test_pred.loc[concat_test_pred['LGBM_Outlier_Pred']==1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Producing better ranking than single LGBM with\n# Private score : 3.61926 (1637, 39.83%), Public score : 3.68346 (329, 8%)\ncreate_file_for_submission(\"lgbm_opt2_130KS_feats_KFold5_outliers_opt1_lr0_01_lgbmc_lr0_02_adj2_avg.csv\",\n card_ids, concat_test_pred['Avg_Pred'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (C) Method 3 : Blend predictions\n> #### 1. LGBM trained all data instances from train dataset\n> #### 2. LGBM trained data instances predicted as non-outliers from train dataset (Outlier Classifier)\n> #### 3. Blending with mean of 2 above models","metadata":{}},{"cell_type":"code","source":"# X_train replaced with only relevant 'train_features', use 'X' instead\n# X_train['outliers'].value_counts()\nX.loc[X['target'] < -33, 'target'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preparation for Layer 3 : Blending model (Mean)\n# Predict outliers in 'target' for train dataset; change values in X_train['outliers']\nX['outliers'] = lgbm_outlier_reg.predict(X[train_features])\n# >= 0.5 will be assign value 1 \nX['outliers'] = X['outliers'].round(0)\nX['outliers'] = X['outliers'].astype(int)\n\n# Predicted 'outliers' are less than 2,207 ('target' < -33)\nX['outliers'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show only predicted outliers\nX.loc[X['outliers']==1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 201554 + 363\n# 201824 + 93 (LGBM classifier with better AUC Score)\nX.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting only the predicted non-outliers (non-assigned outlier with 'outliers' column == 0)\nX_nonoutliers = X.loc[X['outliers'] == 0]\ny_nonoutliers = X.loc[X['outliers'] == 0, 'target']\nX_nonoutliers.shape, X_nonoutliers[train_features].shape, y_nonoutliers.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_nonoutliers[train_features].shape, X_test[train_features].shape, y_nonoutliers.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport time\n\nn_splits = 5\nlgbm_nonoutliers_pred = lgbm_predictions(X_nonoutliers[train_features], X_test[train_features],\n y_nonoutliers, lgbm_params_opt1, n_splits) # lgbm_params_opt2\n\nlgbm_nonoutliers_pred.shape\n# CV RMSE score: 3.54294 (lgbm_params_opt1 with 10K rounds is better than lgbm_params_opt2 with 10K rounds)\n# CV RMSE score: 3.54258 (lgbm_params_opt1 with 10K rounds is best)\n# CV RMSE score: 3.54606 (lgbm_params_opt2 with 10K rounds is better than lgbm_params_opt2 with 1K rounds)\n\n# Early stopping, best iteration is:\n# [1197]\ttraining's rmse: 3.33551\tvalid_1's rmse: 3.67731\n# CV RMSE score: 3.61183 \n# Wall time: 8min 15s","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- #### Blending with mean of 2 above models","metadata":{}},{"cell_type":"code","source":"# Load best ranked predictions from output directory in current session\n# lgbm_pred_df = pd.read_csv('lgbm_opt2_130KS_merid_puramt_date_histmer_rdiff_w2v_installments_K5.csv')\nlgbm_pred_df = pd.read_csv('../input/test-predictions/lgbm_opt2_130KSfeats_K5.csv')\n\nlgbm_nonoutliers_pred_df = pd.DataFrame()\nlgbm_nonoutliers_pred_df[\"card_id\"] = card_ids\nlgbm_nonoutliers_pred_df[\"LGBM_Pred\"] = lgbm_pred_df['target'] # pd.Series(lgbm_predictions)\n\nlgbm_nonoutliers_pred_df['LGBM_NonOutliers_Pred'] = lgbm_nonoutliers_pred\n\n# Lesser predicted outliers than basic classifier\nlgbm_nonoutliers_pred_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_nonoutliers_pred_df['Avg_Pred'] = lgbm_nonoutliers_pred_df[['LGBM_Pred', 'LGBM_NonOutliers_Pred']].mean(axis=1)\nlgbm_nonoutliers_pred_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train['final'] = train['bin_predict'](-33.21928)+(1-train['bin_predict'])train['no_outlier']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using best 130KS features with opt2, KFold5\n# Producing worst ranking than single LGBM with Public score : 3.69549 but better than above outlier methods 1 and 2\ncreate_file_for_submission(\"lgbm_opt2r10K_130KS_feats_KFold5_lgbmc_nonoutliers_avg.csv\",\n card_ids, lgbm_nonoutliers_pred_df['Avg_Pred'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training based on 'target' predictions from layer 1 models with ground truth on train dataset","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (D) Method 4 : Blending with Bayesian Ridge\n> #### 1. LGBM trained all data instances from train dataset\n> #### 2. LGBM trained data instances predicted as non-outliers from train dataset (Outlier Classifier)\n> #### 3. Blending with mean of 2 above models (Method 3)","metadata":{}},{"cell_type":"code","source":"%%time\nfrom sklearn.linear_model import BayesianRidge\n\nbayesridge_clf = BayesianRidge(compute_score=True)\nbayesridge_clf.fit(concat_train_pred, y)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BayesianRidge prediction based on blending\nbayesridge_pred = bayesridge_clf.predict(lgbm_nonoutliers_pred_df[['LGBM_Pred', 'LGBM_NonOutliers_Pred', 'Avg_Pred']])\nbayesridge_pred.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for f in ['feature_1','feature_2','feature_3']:\n# # Setting mean value of the 'outliers' for the input features \n# order_label = X_train.groupby([f])['outliers'].mean()\n# X_train[f] = X_train[f].map(order_label)\n# X_test[f] = X_test[f].map(order_label)\n\n# X_train.loc[0:5, ['feature_1','feature_2','feature_3']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (E) Method 5 : Stacking with Blending\n> #### 1. Load trained single machine learning models of LGBM, XGBR, RF and DL in first layer\n> #### 2. Using Decision Tree Regressor and LGBM in second layer\n> #### 3. Final layer with average blending","metadata":{}},{"cell_type":"code","source":"X[train_features].shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:34:09.625983Z","iopub.execute_input":"2021-10-28T12:34:09.626528Z","iopub.status.idle":"2021-10-28T12:34:09.711635Z","shell.execute_reply.started":"2021-10-28T12:34:09.626491Z","shell.execute_reply":"2021-10-28T12:34:09.710708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -tl ../input/test-predictions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- #### Load from saved predictions from different models","metadata":{}},{"cell_type":"code","source":"# Load model predictions trained with 130 KS features\nlgbm_pred = pd.read_csv('../input/test-predictions/lgbm_opt2_130KSfeats_K5.csv')\nxgbr_pred = pd.read_csv('../input/test-predictions/xgbr_130KSfeats_K5.csv')\nrf_pred = pd.read_csv('../input/test-predictions/rf_130KS_feats.csv')\ndl_pred = pd.read_csv('../input/test-predictions/DL3_v1_130KSfeats_adam.csv')\n\ntest_pred = pd.concat([lgbm_pred['target'], xgbr_pred['target'],\n rf_pred['target'], dl_pred['target']],\n axis=1, keys=['LGBM_Pred', 'XGBR_Pred', 'RF_Pred', 'DL_Pred'])\n\n# dl_preds = pd.read_csv('../input/testpredictions/xgbr_lgbm_dl_79feat.csv')\n# concat_pred_df = pd.concat([xgbr_preds['target'], lgbm_preds['target'], dl_preds['target']],\n# axis=1, keys=['XGBR_Pred', 'LGBM_Pred', 'DL_Pred'])\n\n# concat_pred_df['Avg_Pred'] = concat_pred_df.mean(axis=1)\n\n# xgbr_preds.loc[:5, 'target'], lgbm_preds.loc[:5, 'target'], dl_preds.loc[:5, 'target']\n\n# concat_test_pred.drop('Avg_Pred', axis=1)\n# test_pred = concat_test_pred[['LGBM_1_Pred', 'RF_Pred', 'DL_Pred']]\n# test_pred = concat_test_pred\ntest_pred","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:33:19.348526Z","iopub.execute_input":"2021-10-28T12:33:19.348945Z","iopub.status.idle":"2021-10-28T12:33:19.99137Z","shell.execute_reply.started":"2021-10-28T12:33:19.348912Z","shell.execute_reply":"2021-10-28T12:33:19.989755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the saved Light Gradient Boosting model\nlgbm_reg = lgb.Booster(model_file='../input/test-predictions/lgbm_optmodel2_130KSfeats_K5_3.68718.txt') # init model\nlgbm_reg.params","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:34:13.302286Z","iopub.execute_input":"2021-10-28T12:34:13.302972Z","iopub.status.idle":"2021-10-28T12:34:15.263363Z","shell.execute_reply.started":"2021-10-28T12:34:13.302873Z","shell.execute_reply":"2021-10-28T12:34:15.262386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- #### Load trained single machine learning models of LGBM, XGBR, RF and DL in first layer\n> #### (from shared Kaggle dataset, preloaded in Kaggle Notebook, please contact Jit Seah to have access)","metadata":{}},{"cell_type":"code","source":"# For loading XGBR model\nimport os \nimport xgboost as xgb\n\n#### If XGBRegressor model exists, load it\nxgbr_model_file = '../input/test-predictions/xgbr_130KSfeats_K5.json'\n\nif os.path.exists(xgbr_model_file):\n # xgb_Classifier.save_model and load_model give an \"le\" error when trying to obtain score\n # Retrieve model file if exist so that training do not need to done\n xgbr_model = xgb.XGBRegressor()\n print(\"Unpickling existing XGBRegressor model...\")\n xgbr_model.load_model(xgbr_model_file)\n \n # xgb_model.get_xgb_params()\n print(\"Loaded model :\\n\", xgbr_model)\n print(\"with type\\n\", type(xgbr_model))\n \n# Check if there is any difference in the model saved and loaded predictions\n# There are differences in predictions from loaded model\nxgbr_predictions_new = xgbr_model.predict(X_test[train_features])\npred_diff = np.setdiff1d(xgbr_pred['target'], xgbr_predictions_new)\nlen(pred_diff), xgbr_pred['target'].shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:34:37.132448Z","iopub.execute_input":"2021-10-28T12:34:37.133261Z","iopub.status.idle":"2021-10-28T12:34:42.023942Z","shell.execute_reply.started":"2021-10-28T12:34:37.133203Z","shell.execute_reply":"2021-10-28T12:34:42.022513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For loading DL model in h5 format\nfrom keras.models import load_model\n\n# Load the saved Deep Learning model\ndl_model = load_model('../input/test-predictions/DL3_v1_130KSfeats_adam.h5')\ndl_model.summary()\n\n# Check if there is any difference in the model saved and loaded predictions\n# There are NO differences in predictions from loaded model\ndl_predictions_new = dl_model.predict(X_test[train_features])\npred_diff = np.setdiff1d(dl_pred['target'], dl_predictions_new)\nlen(pred_diff), dl_pred['target'].shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:34:43.745832Z","iopub.execute_input":"2021-10-28T12:34:43.746191Z","iopub.status.idle":"2021-10-28T12:34:56.812624Z","shell.execute_reply.started":"2021-10-28T12:34:43.74616Z","shell.execute_reply":"2021-10-28T12:34:56.811497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For loading Random Forest Regressor model\n# forest_reg = load_model_from_picklefile('rf_1KEst_130KS_feats_pickle')\nforest_reg = joblib.load(\"../input/test-predictions/rf_1KEst_130KSfeats.joblib\")\nprint(\"RF params :\", forest_reg.get_params)\n\n# Check if there is any difference in the model saved and loaded predictions\n# There are differences in predictions from loaded model\nrf_predictions_new = forest_reg.predict(X_test[train_features])\npred_diff = np.setdiff1d(rf_pred['target'], rf_predictions_new)\nlen(pred_diff), rf_pred['target'].shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:34:56.814351Z","iopub.execute_input":"2021-10-28T12:34:56.814666Z","iopub.status.idle":"2021-10-28T12:34:58.623823Z","shell.execute_reply.started":"2021-10-28T12:34:56.814636Z","shell.execute_reply":"2021-10-28T12:34:58.622736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.shape, X_train[train_features].shape, X_test[train_features].shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:35:00.711785Z","iopub.execute_input":"2021-10-28T12:35:00.712292Z","iopub.status.idle":"2021-10-28T12:35:00.814949Z","shell.execute_reply.started":"2021-10-28T12:35:00.712235Z","shell.execute_reply":"2021-10-28T12:35:00.813559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 1. Load trained models of LGBM, XGBR, RF and DL in first layer","metadata":{}},{"cell_type":"code","source":"%%time\nlgbm_train_pred = lgbm_reg.predict(X[train_features])\nxgbr_train_pred = xgbr_model.predict(X[train_features])\nrf_train_pred = forest_reg.predict(X[train_features])\ndl_train_pred = dl_model.predict(X[train_features])\n# dtreg_train_pred = tree_reg.predict(X[train_features])\n\nlgbm_train_pred_df = pd.DataFrame(lgbm_train_pred, columns = ['LGBM_Tr_Pred'])\nxgbr_train_pred_df = pd.DataFrame(xgbr_train_pred, columns = ['XGBR_Tr_Pred'])\nrf_train_pred_df = pd.DataFrame(rf_train_pred, columns = ['RF_Tr_Pred'])\ndl_train_pred_df = pd.DataFrame(dl_train_pred, columns = ['DL_Tr_Pred'])\n# dtreg_train_pred_df = pd.DataFrame(dtreg_train_pred, columns = ['DTR_Tr_Pred'])\n\n# xgbr_train_pred_df, lgbm_train_pred_df, rfb_train_pred_df and dl_train_pred_df\ntrain_pred = pd.concat([lgbm_train_pred_df, xgbr_train_pred_df, rf_train_pred_df, dl_train_pred_df],\n axis=1, sort=False)\n\ntrain_pred","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:35:09.414031Z","iopub.execute_input":"2021-10-28T12:35:09.414432Z","iopub.status.idle":"2021-10-28T12:38:13.207928Z","shell.execute_reply.started":"2021-10-28T12:35:09.414395Z","shell.execute_reply":"2021-10-28T12:38:13.207095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pred.shape, test_pred.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:38:13.209113Z","iopub.execute_input":"2021-10-28T12:38:13.209543Z","iopub.status.idle":"2021-10-28T12:38:13.216513Z","shell.execute_reply.started":"2021-10-28T12:38:13.209513Z","shell.execute_reply":"2021-10-28T12:38:13.21504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2. Using Random Forest Regressor and LGBM in second layer","metadata":{}},{"cell_type":"code","source":"# %%time\n# Linear Regression model to have ensemble regression\n# from sklearn.tree import DecisionTreeRegressor\n# Cost : Root Mean Square Error, RMSE\n# from sklearn.metrics import mean_squared_error\n\n# tree_reg = DecisionTreeRegressor()\n# tree_reg.fit(train_pred, y)\n\n# train_pred = tree_reg.predict(train_pred)\n# tree_mse = mean_squared_error(y, train_pred)\n# tree_rmse = np.sqrt(tree_mse)\n# print(\"DecisionTreeRegressor RMSE :\", tree_rmse)\n\n# scores = cross_val_score(tree_reg, concat_train_pred, y,\n# scoring=\"neg_mean_squared_error\", cv=10)\n\n# tree_rmse_scores = np.sqrt(-scores) # opposite of MSE, need to have negative sign\n\n# # Different results when it is executed\n# display_scores(tree_rmse_scores)\n\n# SL2_tree_reg_pred = tree_reg.predict(test_pred)\n# SL2_tree_reg_pred.shape\n\n# Using a trained linear regressor with predicted values from XGBR, LGBM, RF and DL\n# Overfits and giving poor tree reg stacking results, Public score : 4.91414\n# create_file_for_submission(\"lgbm_xgbr_rf_dl_130KSfeat_SL2_tree_reg.csv\", card_ids, SL2_tree_reg_pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Random forest model as part of ensemble regressor\nfrom sklearn.ensemble import RandomForestRegressor\n\n# The default value of n_estimators will change from 10 in version 0.20 to 100 in version 0.22\nSL2_forest_reg = RandomForestRegressor(n_estimators = 1000,\n max_depth = 9, # 15\n min_samples_leaf = 20, # 149\n min_weight_fraction_leaf = 0.2,\n bootstrap = True,\n oob_score = True,\n n_jobs = -1,\n random_state = 100,\n verbose=1)\n\nSL2_forest_reg.fit(train_pred, y)\n\ndata_predictions = SL2_forest_reg.predict(train_pred)\nSL2_forest_reg_mse = mean_squared_error(y, data_predictions)\nSL2_forest_reg_rmse = np.sqrt(SL2_forest_reg_mse)\n\nprint(\"RandomForestRegressor RMSE :\", forest_rmse)","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:44:25.670537Z","iopub.execute_input":"2021-10-28T12:44:25.670966Z","iopub.status.idle":"2021-10-28T12:45:26.960069Z","shell.execute_reply.started":"2021-10-28T12:44:25.670931Z","shell.execute_reply":"2021-10-28T12:45:26.958966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SL2_rf_reg_pred = SL2_forest_reg.predict(test_pred)\nSL2_rf_reg_pred.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:45:26.961708Z","iopub.execute_input":"2021-10-28T12:45:26.96209Z","iopub.status.idle":"2021-10-28T12:45:28.080731Z","shell.execute_reply.started":"2021-10-28T12:45:26.962054Z","shell.execute_reply":"2021-10-28T12:45:28.079278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using a RF regressor with predicted values from XGBR, LGBM, RF and DL\n# Overfits and doing better than tree reg, Public score : 3.93717\ncreate_file_for_submission(\"lgbm_xgbr_rf_dl_130KSfeat_SL2_rf_reg.csv\", card_ids, SL2_rf_reg_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport time\n\nnum_round = 10000\nn_splits=5\n\nfolds = KFold(n_splits=n_splits, shuffle=True, random_state=4590) # 27\noof = np.zeros(len(train_pred))\nSL2_lgbm_pred = np.zeros(len(test_pred))\nstart = time.time()\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_pred.values, y.values)):\n print(\"LGBM fold n°{}\".format(fold_))\n print(\"-\" * 80)\n \n trn_data = lgb.Dataset(train_pred.iloc[trn_idx], # [train_features]\n label=y.iloc[trn_idx])\n val_data = lgb.Dataset(train_pred.iloc[val_idx], # [train_features]\n label=y.iloc[val_idx])\n\n # lgbm_params_opt2 performs better than lgbm_params_opt1 andlgbm_params \n SL2_lgbm_reg = lgb.train(lgbm_params_opt2,\n trn_data,\n num_round,\n valid_sets = [trn_data, val_data],\n verbose_eval=100,\n early_stopping_rounds = 200)\n \n oof[val_idx] = SL2_lgbm_reg.predict(train_pred.iloc[val_idx], # [train_features]\n num_iteration=SL2_lgbm_reg.best_iteration)\n \n fold_importance_df = pd.DataFrame()\n fold_importance_df[\"feature\"] = train_pred.columns\n fold_importance_df[\"importance\"] = SL2_lgbm_reg.feature_importance()\n fold_importance_df[\"fold\"] = fold_ + 1\n feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n \n SL2_lgbm_pred += SL2_lgbm_reg.predict(test_pred,\n num_iteration=SL2_lgbm_reg.best_iteration) / folds.n_splits\n print()\n \n # Save model\n # lgbm_pred_reg.save_model('lgbm_pred_reg_model.txt', num_iteration=bst.best_iteration)\n\nprint(\"CV RMSE score: {:<8.5f}\".format(mean_squared_error(oof, y)**0.5))","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:40:10.193656Z","iopub.execute_input":"2021-10-28T12:40:10.194222Z","iopub.status.idle":"2021-10-28T12:41:46.060249Z","shell.execute_reply.started":"2021-10-28T12:40:10.19418Z","shell.execute_reply":"2021-10-28T12:41:46.059369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Wall time: 28.1 s (using lgbm_params_opt1)\n# CV RMSE score: 3.84539\n# Wall time: 1min 40s (using lgbm_params_opt2)\n# CV RMSE score: 3.84296\n# Wall time: 27.7 s (using lgbm_params)\n# CV RMSE score: 3.84527\n# lgbm_params_opt2 gives better public score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Early stopping, best iteration is:\n# [3656]\ttraining's rmse: 2.27884\tvalid_1's rmse: 2.28246\n# CV RMSE score: 2.33106\n# Wall time: 5min 6s","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pred.shape, SL2_lgbm_pred.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using a LGBM with predicted values from XGBR, LGBM, RF and DL\n# Overfits and giving poor LGBM stacking results with opt2, Public score : 3.92293 (3.87041)\n# Overfits and giving poor LGBM stacking results with opt1, Public score : 3.93402\n# Overfits and giving poor LGBM stacking results with lgbm_params, Public score : 3.93414\ncreate_file_for_submission(\"lgbm_xgbr_rf_dl_130KSfeat_SL2_lgbm.csv\", card_ids, SL2_lgbm_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom sklearn.linear_model import BayesianRidge\n\nbayesridge_clf = BayesianRidge(compute_score=True)\nbayesridge_clf.fit(train_pred, y)\n\nSL2_bayesridge_pred = bayesridge_clf.predict(test_pred)\nSL2_bayesridge_pred.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:09:26.919796Z","iopub.execute_input":"2021-10-28T13:09:26.920207Z","iopub.status.idle":"2021-10-28T13:09:27.035003Z","shell.execute_reply.started":"2021-10-28T13:09:26.920175Z","shell.execute_reply":"2021-10-28T13:09:27.032232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using a trained linear regressor with predicted values from XGBR, LGBM, RF and DL\n# Overfits but giving better stacking results than stack 2 LGBM and Tree Reg, Public score : 3.72625\ncreate_file_for_submission(\"lgbm_xgbr_rf_dl_130KSfeat_SL2_bayes_ridge.csv\", card_ids, SL2_bayesridge_pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 3. Final layer with average blending","metadata":{}},{"cell_type":"code","source":"# Stacking layer No. 3\nSL3_rf_reg_test_pred = pd.DataFrame(SL2_rf_reg_pred, columns = ['ESM_RF_Pred'])\nSL3_lgbm_test_pred = pd.DataFrame(SL2_lgbm_pred, columns = ['ESM_LGBM_Pred'])\nSL3_bayesr_test_pred = pd.DataFrame(SL2_bayesridge_pred, columns = ['ESM_BAYESR_Pred'])\n\nensemble_test_pred = pd.concat([SL3_rf_reg_test_pred, SL3_lgbm_test_pred, SL3_bayesr_test_pred],\n axis=1, sort=False)\nensemble_test_pred['Avg_Blend_Pred'] = ensemble_test_pred.mean(axis=1)\n\nensemble_test_pred","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:12:48.549581Z","iopub.execute_input":"2021-10-28T13:12:48.549991Z","iopub.status.idle":"2021-10-28T13:12:48.57677Z","shell.execute_reply.started":"2021-10-28T13:12:48.54996Z","shell.execute_reply":"2021-10-28T13:12:48.575671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensemble_test_pred['Avg_Blend_Pred'].shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:16:19.372435Z","iopub.execute_input":"2021-10-28T13:16:19.372833Z","iopub.status.idle":"2021-10-28T13:16:19.379288Z","shell.execute_reply.started":"2021-10-28T13:16:19.372793Z","shell.execute_reply":"2021-10-28T13:16:19.378062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using a trained linear regressor with predicted values from XGBR, LGBM, RF and DL\n# Overfits but giving better stacking results than stack 2 Tree Reg, Public score : 3.92238\ncreate_file_for_submission(\"lgbm_xgbr_rf_dl_130KSfeat_SL3_avg_blend.csv\", card_ids,\n ensemble_test_pred['Avg_Blend_Pred'])","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:13:27.513234Z","iopub.execute_input":"2021-10-28T13:13:27.513634Z","iopub.status.idle":"2021-10-28T13:13:28.043568Z","shell.execute_reply.started":"2021-10-28T13:13:27.513583Z","shell.execute_reply":"2021-10-28T13:13:28.042471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (F) Method 6 : Stacking with BayesianRidge in final Blending\n> #### 1. Load trained single machine learning models of LGBM, XGBR, RF and DL in first layer\n> #### 2. Using Random Forest Regressor and LGBM in second layer\n> #### 3. Final layer with BayesianRidge","metadata":{}},{"cell_type":"code","source":"train_pred.shape, test_pred.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:57:10.686137Z","iopub.execute_input":"2021-10-28T12:57:10.686528Z","iopub.status.idle":"2021-10-28T12:57:10.695149Z","shell.execute_reply.started":"2021-10-28T12:57:10.686495Z","shell.execute_reply":"2021-10-28T12:57:10.694119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SL2_rf_train_pred = SL2_forest_reg.predict(train_pred)\nSL2_lgbm_train_pred = SL2_lgbm_reg.predict(train_pred)\n\nSL2_rf_train_pred_df = pd.DataFrame(SL2_rf_train_pred, columns = ['SL2_RF_Tr_Pred'])\nSL2_lgbm_train_pred_df = pd.DataFrame(SL2_lgbm_train_pred, columns = ['SL2_LGBM_Tr_Pred'])\n\nSL2_train_pred = pd.concat([SL2_lgbm_train_pred_df, SL2_rf_train_pred_df],\n axis=1, sort=False)\n\nSL2_train_pred","metadata":{"execution":{"iopub.status.busy":"2021-10-28T12:56:24.077501Z","iopub.execute_input":"2021-10-28T12:56:24.077903Z","iopub.status.idle":"2021-10-28T12:56:39.43883Z","shell.execute_reply.started":"2021-10-28T12:56:24.077871Z","shell.execute_reply":"2021-10-28T12:56:39.43727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SL2_rf_test_pred = pd.DataFrame(SL2_rf_reg_pred, columns = ['SL2_RF_Pred'])\nSL2_lgbm_test_pred = pd.DataFrame(SL2_lgbm_pred, columns = ['SL2_LGBM_Pred'])\n\nSL2_test_pred = pd.concat([SL2_rf_test_pred, SL2_lgbm_test_pred], axis=1, sort=False)\nSL2_test_pred","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:03:00.420393Z","iopub.execute_input":"2021-10-28T13:03:00.420861Z","iopub.status.idle":"2021-10-28T13:03:00.439916Z","shell.execute_reply.started":"2021-10-28T13:03:00.420821Z","shell.execute_reply":"2021-10-28T13:03:00.438714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SL2_train_pred.shape, SL2_test_pred.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:03:11.414878Z","iopub.execute_input":"2021-10-28T13:03:11.415418Z","iopub.status.idle":"2021-10-28T13:03:11.422561Z","shell.execute_reply.started":"2021-10-28T13:03:11.415385Z","shell.execute_reply":"2021-10-28T13:03:11.42174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Stacking layer No. 3\nSL3_test_pred = SL2_test_pred\nSL3_test_pred['Avg_Blend_Pred'] = SL2_test_pred.mean(axis=1)\nSL3_test_pred","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:23:44.447135Z","iopub.execute_input":"2021-10-28T13:23:44.447694Z","iopub.status.idle":"2021-10-28T13:23:44.467055Z","shell.execute_reply.started":"2021-10-28T13:23:44.447647Z","shell.execute_reply":"2021-10-28T13:23:44.465863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using a trained linear regressor with predicted values from XGBR, LGBM, RF and DL\n# Overfits but giving better stacking results than stack 2 Tree Reg, Public score : 3.91828\ncreate_file_for_submission(\"lgbm_xgbr_rf_dl_130KSfeat_SL2_rf_lgbm_SL3_avg_blend.csv\", card_ids,\n SL3_test_pred['Avg_Blend_Pred'])","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:25:22.268532Z","iopub.execute_input":"2021-10-28T13:25:22.268937Z","iopub.status.idle":"2021-10-28T13:25:22.808173Z","shell.execute_reply.started":"2021-10-28T13:25:22.268905Z","shell.execute_reply":"2021-10-28T13:25:22.806748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom sklearn.linear_model import BayesianRidge\n\n# Stacking layer No.3 : BayesianRidge on layer 2 models (RF, LGBM)\nbayesridge_clf = BayesianRidge(compute_score=True)\nbayesridge_clf.fit(SL2_train_pred, y)\n\nSL3_bayesridge_pred = bayesridge_clf.predict(SL2_test_pred)\nSL3_bayesridge_pred.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:04:12.487432Z","iopub.execute_input":"2021-10-28T13:04:12.487796Z","iopub.status.idle":"2021-10-28T13:04:12.581685Z","shell.execute_reply.started":"2021-10-28T13:04:12.487766Z","shell.execute_reply":"2021-10-28T13:04:12.580235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using a trained linear regressor with predicted values from XGBR, LGBM, RF and DL\n# Overfits but giving better stacking results than stack 2 Tree Reg, Public score : 3.91297\ncreate_file_for_submission(\"lgbm_xgbr_rf_dl_130KSfeat_SL3_bayesridge.csv\", card_ids, SL3_bayesridge_pred)","metadata":{"execution":{"iopub.status.busy":"2021-10-28T13:04:30.759818Z","iopub.execute_input":"2021-10-28T13:04:30.76021Z","iopub.status.idle":"2021-10-28T13:04:31.302327Z","shell.execute_reply.started":"2021-10-28T13:04:30.760176Z","shell.execute_reply":"2021-10-28T13:04:31.301157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part 3 : Single Baseline Models","metadata":{}},{"cell_type":"code","source":"list_discarded","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- #### Exclude unwanted features for training without recreating the merged data","metadata":{}},{"cell_type":"code","source":"X_train[train_features].shape, X_test[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove features in attempt to get better CV RMSE scores\nprint(\"No. of train_features :\", len(train_features))\n\nold_list_discarded = ['hist_purchase_amount_max', 'hist_category_3_sum_purchase_amount_mean', 'hist_installments_mean']\nfeatures_to_remove = list_discarded\n# features_to_remove = ['new_merchant_purchase_amount_sum', 'new_merchant_merchants_most_recent_purchases_range_min_max',\n# 'new_merchant_merchants_avg_purchases_lag3_sum_min', 'new_merchant_merchants_avg_sales_lag6_mean_max',\n# 'new_merchant_merchants_most_recent_sales_range_max_min', 'new_merchant_w2v_new_state_id_0_std',\n# 'new_merchant_card_id_size', 'new_merchant_w2v_new_state_id_1_std',\n# 'new_merchant_merchants_avg_sales_lag3_mean_max', 'new_merchant_merchants_avg_purchases_lag6_sum_max']\n\ntrain_features = list(set(train_features) - set(features_to_remove))\n\nprint(\"No. of train_features after removing some low important features :\", len(train_features))\n\nlen(set(features_to_remove)), len(set(train_features))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cost : Root Mean Square Error, RMSE\nfrom sklearn.metrics import mean_squared_error\n# Better Evaluation using cross-validation\nfrom sklearn.model_selection import cross_val_score\n\ndef display_scores(scores):\n print(\"Scores:\", scores)\n print(\"Mean:\", scores.mean())\n print(\"Standard Deviation:\", scores.std())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (A) Random Forest Regressor","metadata":{}},{"cell_type":"code","source":"X_train[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Ensemble Learning : RandomForestRegressor\nfrom sklearn.ensemble import RandomForestRegressor\n\n# The default value of n_estimators will change from 10 in version 0.20 to 100 in version 0.22\nforest_reg = RandomForestRegressor(n_estimators = 1000, # 1000\n max_depth = 9, # 15\n min_samples_leaf = 20, # 149\n min_weight_fraction_leaf = 0.2,\n bootstrap = True,\n oob_score = True, n_jobs = -1,\n random_state = 100,\n verbose=1)\n\nforest_reg.fit(X_train[train_features], X_train['target'])\n\ndata_predictions = forest_reg.predict(X_train[train_features])\nforest_mse = mean_squared_error(X_train['target'], data_predictions)\nforest_rmse = np.sqrt(forest_mse)\n\nprint(\"RandomForestRegressor RMSE :\", forest_rmse)\n\n# forest_scores = cross_val_score(forest_reg, X_train[train_features], X_train['target'],\n# scoring=\"neg_mean_squared_error\", cv=10)\n\n# forest_rmse_scores = np.sqrt(-forest_scores) # opposite of MSE, need to have negative sign\n\n# display_scores(forest_rmse_scores)\n\n# Scores: [3.90089189 4.04644846 3.99770616 3.98337336 4.00308421 3.89694652\n# 3.9277469 4.12725579 4.00782351 4.04242215]\n# Mean: 3.993369895502748\n# Standard Deviation: 0.06762168624862287\n# Wall time: 14min 53s\n\n# RandomForestRegressor RMSE : 3.7898758159311647 (194 features with PriSc: 3.75530, PubSc: 3.86015)\n# RandomForestRegressor RMSE : 3.7979328113935664 (130KS features with PriSc: 3.76409, PubSc: 3.86907)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# forest_reg.oob_score_\nforest_reg.score(X_train[train_features], X_train['target'])\n# max_depth = 9 or 15 RandomForestRegressor gives same RMSE : 3.7979328113935664\n# forest_reg.score is 0.02711288566923864","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_predictions = forest_reg.predict(X_test[train_features])\nrf_predictions.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the model\n# save_model_to_picklefile('rf_1KEst_130KS_feats_pickle', forest_reg)\njoblib.dump(forest_reg, \"rf_1KEst_130KSfeats.joblib\")\ndel forest_reg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the saved model\n# forest_reg = load_model_from_picklefile('rf_1KEst_130KS_feats_pickle')\nforest_reg = joblib.load(\"rf_1KEst_130KS_feats.joblib\")\nprint(\"RF params :\", forest_reg.get_params)\n\n# Check if there is any difference in the model saved and loaded predictions\n# There are differences in predictions from loaded model\nrf_predictions_new = forest_reg.predict(X_test[train_features])\npred_diff = np.setdiff1d(rf_predictions, rf_predictions_new)\n\nlen(pred_diff)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_file_for_submission(\"rf_130KS_feats.csv\", X_test[\"card_id\"], rf_predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Search the best combination of hyperparameter values\nfrom sklearn.model_selection import GridSearchCV\n\nparam_grid = [\n {'n_estimators': [100, 200], 'max_features': [80, 120, X_train[train_features].shape[1]]}, # 'max_features': [2, 4, 6, 8]\n {'bootstrap': [False], 'n_estimators': [100, 200], 'max_features': [100, X_train[train_features].shape[1]]}, # 'max_features': [2, 3, 4]\n]\n\n# The default value of n_estimators will change from 10 in version 0.20 to 100 in version 0.22\nforest_reg = RandomForestRegressor(n_estimators=200) # no more warning on default 'n_estimators' value\n\ngrid_search = GridSearchCV(forest_reg, param_grid, cv=7, scoring=\"neg_mean_squared_error\")\n\ngrid_search.fit(X_train[train_features], y)\n\n# CPU times: user 46min 45s, sys: 4.79 s, total: 46min 50s\n# Wall time: 46min 50s\n\n# GridSearchCV(cv=5, estimator=RandomForestRegressor(n_estimators=10),\n# param_grid=[{'max_features': [10, 15, 20],\n# 'n_estimators': [3, 10, 30]},\n# {'bootstrap': [False], 'max_features': [9, 11, 17],\n# 'n_estimators': [3, 10]}],\n# scoring='neg_mean_squared_error')\n\n# grid_search.best_params_\n# {'max_features': 15, 'n_estimators': 30}\n\n# GridSearchCV(cv=5, estimator=RandomForestRegressor(n_estimators=10),\n# param_grid=[{'max_features': [10, 15, 20, 30, 37],\n# 'n_estimators': [3, 10, 30]},\n# {'bootstrap': [False],\n# 'max_features': [9, 11, 17, 27, 31],\n# 'n_estimators': [3, 10]}],\n# scoring='neg_mean_squared_error')\n\n# grid_search.best_params_\n# {'max_features': 10, 'n_estimators': 30}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search.best_params_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search.best_estimator_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_result = grid_search.cv_results_\nfor mean_score, params in zip(cv_result[\"mean_test_score\"], cv_result[\"params\"]):\n print(np.sqrt(-mean_score), params) # negative mean_score\n \n# RMSE : 3.745822393200406 {'max_features': 10, 'n_estimators': 30} with 85 features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analyze the Best Models and their Errors\nfeature_importances = grid_search.best_estimator_.feature_importances_\nfeature_importances","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(feature_importances)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using GridSearchCV to obtain final model with best estimator\n# final_model = grid_search.best_estimator_\nrf_best = grid_search.best_estimator_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"brf_predictions = rf_best.predict(X_test[train_features])\nbrf_predictions.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_file_for_submission(\"brf_138_feats.csv\", card_ids, brf_predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.linear_model import LinearRegression\n\n# # Linear Regression model\n# linear_reg = LinearRegression()\n# linear_reg.fit(X_train, data_labels)\n\n# # first_batchdata = X_train.iloc[:5]\n# # first_batchlabels = data_labels.iloc[:5]\n# # print(\"Predictions: \", linear_reg.predict(first_batchdata))\n# # print(\"Labels: \", list(first_batchlabels))\n\n# data_predictions = linear_reg.predict(X_train)\n# linear_mse = mean_squared_error(data_labels, data_predictions)\n# linear_rmse = np.sqrt(linear_mse)\n\n# linear_rmse # underfitting data; high bias\n\n# %%time\n# # More Powerful Algorithm : DecisionTreeRegressor\n# from sklearn.tree import DecisionTreeRegressor\n\n# tree_reg = DecisionTreeRegressor()\n# tree_reg.fit(X_train[train_features], X_train['target'])\n\n# tree_reg_predictions = tree_reg.predict(X_train[train_features])\n# tree_mse = mean_squared_error(X_train['target'], tree_reg_predictions)\n# tree_rmse = np.sqrt(tree_mse)\n\n# print(\"DecisionTreeRegressor RMSE :\", tree_rmse)\n\n# scores = cross_val_score(tree_reg, X_train, data_labels,\n# scoring=\"neg_mean_squared_error\", cv=10)\n\n# tree_rmse_scores = np.sqrt(-scores) # opposite of MSE, need to have negative sign\n\n# # Different results when it is executed\n# display_scores(tree_rmse_scores)\n\n# card_ids = X_test[\"card_id\"].copy()\n# tree_reg_predictions = tree_reg.predict(X_test[train_features])\n\n# create_file_for_submission(\"dtreereg_85_feats.csv\", card_ids, tree_reg_predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (B) eXtreme Gradient Boosting Regressor (XGBR)\n> https://xgboost.readthedocs.io/en/latest/parameter.html#","metadata":{}},{"cell_type":"code","source":"X_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for missing values : X_train.isna().any()\ncheck_missing_values(X_train), check_missing_values(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data for training and prediction\nX = X_train.copy()\ny = X['target']\ncard_ids = X_test[\"card_id\"].copy()\n\nX.shape, y.shape, X_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for intersection between train features and missing values\nintersect_features = set(train_features).intersection(check_missing_values(X_test)) # df_train\nprint(\"Common features in train and missing value test features (should be 0): {}\".format(len(intersect_features)))\nintersect_features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Difference in card_ids\ncol_difference = set(X.columns).symmetric_difference(X_test.columns)\nprint(\"Difference in train and test features: {}\".format(len(col_difference)))\ncol_difference","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.isinf(X[train_features]).any().value_counts()\n\n# print(\"printing column name where infinity is present\")\n# col_name = X[train_features].columns.to_series()[np.isinf(X[train_features]).any()]\n# print(col_name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.isinf(X_test[train_features]).any().value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, y.shape, X_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n\nimport xgboost as xgb\nnp.random.seed(4590) # 2728\nn_splits = 5\n\nkfolds = KFold(n_splits=n_splits, shuffle=True, random_state=4590) # 2018\n\nprint(\"kfolds :\", kfolds)\n# Make importance dataframe\nimportances = pd.DataFrame()\n\noof_preds = np.zeros(X.shape[0])\nsub_preds = np.zeros(X_test.shape[0])\n\n## timeit will cause problem with importance\nfor n_fold, (trn_idx, val_idx) in enumerate(kfolds.split(X[train_features], y)): # X\n X_train_kf, y_train_kf = X[train_features].iloc[trn_idx], y.iloc[trn_idx]\n X_valid_kf, y_valid_kf = X[train_features].iloc[val_idx], y.iloc[val_idx]\n \n print(\"XGBR fold :\", n_fold)\n print(\"=\" * 80)\n \n # XGBoost Regressor estimator\n xgb_model = xgb.XGBRegressor(\n max_depth = 10, # 15\n learning_rate = 0.01,\n n_estimators = 1000,\n subsample = .9,\n colsample_bylevel = .9,\n colsample_bytree = .9,\n min_child_weight= .9,\n gamma = 0,\n random_state = 4590, # 100\n booster = 'gbtree',\n objective = 'reg:squarederror' # 'reg:linear' deprecated\n )\n \n # Training\n xgb_model.fit(\n X_train_kf, y_train_kf,\n eval_set=[(X_train_kf, y_train_kf), (X_valid_kf, y_valid_kf)],\n verbose=True, eval_metric='rmse',\n early_stopping_rounds=100\n )\n \n # Feature importance\n imp_df = pd.DataFrame()\n imp_df['feature'] = train_features\n imp_df['gain'] = xgb_model.feature_importances_\n imp_df['fold'] = n_fold + 1\n importances = pd.concat([importances, imp_df], axis=0, sort=False)\n \n oof_preds[val_idx] = xgb_model.predict(X_valid_kf)\n test_preds = xgb_model.predict(X_test[train_features])\n sub_preds += test_preds / kfolds.n_splits\n \n print(\"Next fold :\", n_fold+1)\n print()\n \nprint(\"Final RMSE : \", np.sqrt(mean_squared_error(y, oof_preds)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Validations with RMSE\n> [391]\tvalidation_0-rmse:1.51994\tvalidation_1-rmse:3.73630
\n> Final RMSE : 3.6854681990466274
\n\n> [385]\tvalidation_0-rmse:1.61745\tvalidation_1-rmse:3.73726
\n> Final RMSE : 3.687870975295424
\n> Wall time: 1h 19min 11s
\n\n> [445]\tvalidation_0-rmse:1.52606\tvalidation_1-rmse:3.72377
\n> KFold : 5
\n> Final RMSE : 3.6762817708713555
\n> Wall time: 2h 30min 50s
\n\n> 130KS features\n> [709]\tvalidation_0-rmse:2.45425\tvalidation_1-rmse:3.64890
\n> Next fold : 5
\n> Final RMSE : 3.662147988178499
\n> Wall time: 1h 14min 59s
\n> Private score 3.62528 ; Public score : 3.69528
\n--
\n**Reference timing:**
\n[721]\tvalidation_0-rmse:1.21498\tvalidation_1-rmse:3.88689
\nNext fold : 17
\nFinal RMSE : 3.6687819164983986
\n**Wall time: 6h 23min 7s**
","metadata":{}},{"cell_type":"code","source":"X_train[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Save trained ML models","metadata":{}},{"cell_type":"code","source":"# https://xgboost.readthedocs.io/en/latest/python/python_api.html?highlight=save_model#xgboost.XGBRegressor.save_model\n# enable JSON format support for model IO (saving only the trees and objective)\nxgb_model.save_model('xgbr_130KSfeats_K5.json')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pickling files with pickle.dump, then the model may not be accessible in later versions of XGBoost\nprint(\"Pickling one of the XGBRegressor model...\")\nfilename = './XGBRegressor_model_pickle'\nsave_model_to_picklefile(filename, xgb_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Feature Importances","metadata":{}},{"cell_type":"code","source":"importances['gain_log'] = importances['gain']\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(16, 24))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))\n\nplt.title('XGBRegressor Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('xgbr_importances_126feats_K17.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Length of submission\nlen(sub_preds), n_fold","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgbr_predictions = sub_preds\nxgbr_predictions.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_file_for_submission(\"xgbr_130KSfeats_K5.csv\", card_ids, xgbr_predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os \n\n#### If XGBRegressor model exists, load it\nxgbr_model_file = '../input/testpredictions/xgbr_130KSfeats_K5.json'\n\nif os.path.exists(xgbr_model_file):\n # xgb_Classifier.save_model and load_model give an \"le\" error when trying to obtain score\n # Retrieve model file if exist so that training do not need to done\n xgb_model = xgb.XGBRegressor()\n print(\"Unpickling existing XGBRegressor model...\")\n xgb_model.load_model(xgbr_model_file)\n \n # xgb_model.get_xgb_params()\n print(\"Loaded model :\\n\", xgb_model)\n print(\"with type\\n\", type(xgb_model))\n \n# Check if there is any difference in the model saved and loaded predictions\n# There are differences in predictions from loaded model\nxgbr_predictions_new = xgb_model.predict(X_test[train_features])\npred_diff = np.setdiff1d(xgbr_predictions, xgbr_predictions_new)\n\nlen(pred_diff), xgbr_predictions.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### If XGBRegressor model exists, load it\nretrieve_xgbr_file = './XGBRegressor_model_pickle'\n\nif os.path.exists(retrieve_xgbr_file):\n # xgb_Classifier.save_model and load_model give an \"le\" error when trying to obtain score\n # Unpickling saved binary file if exist so that training do not need to done\n loaded_XGBRegressor = load_model_from_picklefile(retrieve_xgbr_file)\n print(\"Unpickling existing XGBRegressor model...\")\n print(\"Loaded model :\\n\", loaded_XGBRegressor)\n print(\"with type\\n\", type(loaded_XGBRegressor))\n \n# Must test with load_model()\nxgbr_predictions_new = xgb_model.predict(X_test[train_features])\npred_diff = np.setdiff1d(xgbr_predictions, xgbr_predictions_new)\n\nlen(pred_diff), xgbr_predictions.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (C) Light Gradient Boosting Machine (LGBM)","metadata":{}},{"cell_type":"markdown","source":"- #### Exclude interim features to avoid strongly correlated features","metadata":{}},{"cell_type":"code","source":"# len(feature_importance_df)\n# feature_importance_df[[\"feature\", \"importance\"]].sort_values(by=\"importance\", ascending=False)[570:580]\n\n# remove features to get better CV RMSE scores\nprint(\"No. of train_features :\", len(train_features))\n# features_to_remove = ['new_merchant_merchants_category_1_min_min', 'new_merchant_merchants_category_1_min_max',\n# 'new_merchant_merchants_category_1_max_min', 'new_merchant_merchants_category_1_max_max']\n\n# 'new_merchant_merchants_active_months_lag12_min_min', 'new_merchant_merchants_avg_sales_lag3_max_min' was left out without a comma\n# gives a better PB : 3.69303\n# 'new_merchant_merchants_active_months_lag12_sum_min', 'new_merchant_merchants_active_months_lag12_min_min',\n# 'new_merchant_merchants_avg_sales_lag3_max_min', 'new_merchant_merchants_avg_sales_lag6_max_max',\n# 'new_merchant_merchants_avg_sales_lag6_max_min', 'new_merchant_merchants_active_months_lag6_sum_mean',\n# 'new_merchant_merchants_avg_sales_lag12_max_max', 'new_merchant_merchants_avg_sales_lag3_max_max'\n\n# ['new_merchant_merchants_merchant_category_id_std_min', 'new_merchant_merchants_merchant_category_id_std_std',\n# 'new_merchant_merchants_merchant_category_id_std_max',\n# 'hist_purchase_date_max_month', 'hist_purchase_date_min_month',\n# 'new_merchant_purchase_date_max_month', 'new_merchant_purchase_date_min_month']\n\n# 'hist_purchase_date_dayofweek_nunique',\n\n# Exclude 'hist_category_1', 'hist_purchase_date_int', 'hist_card_purchase_date_max'\n# 'new_card_purchase_date_max', 'new_purchase_date_int'\n# ['hist_purchase_amount_max', 'hist_category_3_sum_purchase_amount_mean'] from list_discarded\n\n# features_to_remove = ['hist_sum_category_1_1_purchase_amount','hist_purchase_date_year_nunique',\n# 'new_merchant_purchase_date_year_nunique',\n \n# 'new_merchant_card_id_size', \n \n# # Did not give better rank with lesser features (148 to 143)\n# 'hist_purchase_date_int', # removed 'new_purchase_date_int' in list_discarded\n# 'new_merchant_purchase_date_month_nunique', 'new_merchant_purchase_date_year_mean',\n# 'new_merchant_purchase_date_dayofweek_nunique']\n\n# Remove features with low importance makes ranking worst than just removing KS list_discarded\nfeatures_to_remove = ['hist_sum_category_1_1_purchase_amount','hist_card_mean_purchase_amount_new',\n 'new_merchant_card_min_purchase_amount_new', 'hist_card_sum_purchase_amount_new',\n 'hist_card_min_purchase_amount_new', 'hist_category_1',\n 'new_merchant_card_sum_purchase_amount_new', 'new_merchant_purchase_amount_new',\n 'hist_sum_category_1_purchase_amount', 'new_merchant_card_median_purchase_amount_new']#'new_merchant_merchants_merchant_category_id_std_min',\n #'new_merchant_merchants_merchant_category_id_std_std',\n #'new_merchant_merchants_merchant_category_id_std_max',\n # Present in 'X_train_137_KSfeats.csv' and was identified to be discarded\n #'hist_w2v_hist_merchant_id_1_std']\n\ntrain_features = list(set(train_features) - set(features_to_remove))\n\nprint(\"No. of train_features after removing some low important features :\", len(train_features))\n\nlen(set(features_to_remove)), len(set(train_features))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reference:\n# old_list_discarded = ['hist_purchase_amount_max', 'hist_category_3_sum_purchase_amount_mean', 'hist_installments_mean']\n# current_list_discarded = ['hist_purchase_amount_max', 'hist_category_3_sum_purchase_amount_mean',\n# 'hist_w2v_hist_merchant_id_0_std', 'hist_installments_mean']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using LGBM hyperparameters after tuning","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\n\n# StratifiedKFold for binary or multiclass\n# from sklearn.model_selection import StratifiedKFold\n# import warnings\n# warnings.filterwarnings('ignore')\nnp.random.seed(4590)\n\n# Optimised lgbm params : improvement from fine tuning\nlgbm_params_opt1 = {'num_leaves': 127,\n 'min_data_in_leaf': 148, # 'min_child_samples': 89 ignored\n 'objective':'regression',\n 'max_depth': 10,\n 'learning_rate': 0.005,\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.7202,\n \"bagging_freq\": 1,\n \"bagging_fraction\": 0.8125,\n \"bagging_seed\": 11,\n \"metric\": 'rmse',\n \"lambda_l1\": 0.3468,\n \"random_state\": 4590, # 133,\n \"verbosity\": -1}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import KFold\nnp.random.seed(4590)\n\n# Optimised lgbm params : check if there is improvement\n# Version 41 (177 features) with the following hyperparameter tuning - \nlgbm_params_opt2 = {'num_leaves':62,\n 'min_data_in_leaf': 20,\n 'objective':'regression',\n 'max_depth': 8,\n 'learning_rate': 0.001,\n # \"min_child_samples\": 20,\n \"boosting\": \"gbdt\",\n \"feature_fraction\": 0.9,\n \"bagging_freq\": 1,\n \"bagging_fraction\": 0.9,\n \"bagging_seed\": 11,\n \"metric\": 'rmse',\n \"lambda_l1\": 0.1,\n \"verbosity\": -1,\n # \"nthread\": 4,\n \"random_state\": 4590}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X[['feature_1', 'feature_2', 'feature_3']].head(2)\nX[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport time\ncategorical_feats = ['feature_1', 'feature_2', 'feature_3'] # 'feature_1'\nnum_round = 10000\nn_splits= 7 # 5\n\n# folds = KFold(n_splits=17, shuffle=True, random_state=15)\nfolds = KFold(n_splits=n_splits, shuffle=True, random_state=4590)\n\noof = np.zeros(len(X[train_features]))\nlgbm_predictions = np.zeros(len(X_test[train_features]))\nstart = time.time()\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X[train_features].values, y.values)):\n print(\"LGBM fold n°{}\".format(fold_))\n print(\"-\" * 80)\n \n trn_data = lgb.Dataset(X.iloc[trn_idx][train_features],\n label=y.iloc[trn_idx],\n categorical_feature=categorical_feats)\n val_data = lgb.Dataset(X.iloc[val_idx][train_features],\n label=y.iloc[val_idx],\n categorical_feature=categorical_feats)\n\n # num_round = 10000\n lgbm_reg = lgb.train(lgbm_params_opt2, # lgbm_params\n trn_data,\n num_round,\n valid_sets = [trn_data, val_data],\n verbose_eval=100,\n early_stopping_rounds = 200)\n \n oof[val_idx] = lgbm_reg.predict(X.iloc[val_idx][train_features], num_iteration=lgbm_reg.best_iteration)\n \n fold_importance_df = pd.DataFrame()\n fold_importance_df[\"feature\"] = train_features\n fold_importance_df[\"importance\"] = lgbm_reg.feature_importance()\n fold_importance_df[\"fold\"] = fold_ + 1\n feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n \n lgbm_predictions += lgbm_reg.predict(X_test[train_features], num_iteration=lgbm_reg.best_iteration) / folds.n_splits\n print()\n \n # Save model\n # lgbm_reg.save_model('lgbm_model.txt', num_iteration=bst.best_iteration)\n\nprint(\"CV RMSE score: {:<8.5f}\".format(mean_squared_error(oof, y)**0.5))\n\nX[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_reg.save_model('lgbm_optmodel2_130KSfeats_K5_3.68718.txt', num_iteration=lgbm_reg.best_iteration)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observing feature importances\n- Feature selection","metadata":{}},{"cell_type":"code","source":"# Top features with max importance values\ntop_most_impt_features = (feature_importance_df[[\"feature\", \"importance\"]]\n .groupby(\"feature\")\n .max()\n .sort_values(by=\"importance\", ascending=False)[:21]).reset_index(drop=False)\n\n# top_most_impt_features['feature']\ntop_most_impt_features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n .groupby(\"feature\")\n .mean()\n .sort_values(by=\"importance\", ascending=False).index) # [:1000].index for top 1000 features\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n y=\"feature\",\n data=best_features.sort_values(by=\"importance\", ascending=False))\n\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_opt_130KS_feats_K5_importances.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# features_correlation = pd.DataFrame(top_most_impt_features, columns=top_most_impt_features['feature'])\n\n# colormap = plt.cm.RdBu\n# plt.figure(figsize=(10,8))\n# plt.title('Pearson Correlation of train features', y=1.05, size=15)\n# sns.heatmap(features_correlation.astype(float).corr(),linewidths=0.1,vmax=1.0, \n# square=True, cmap=colormap, linecolor='white', annot=True)\n\n# plt.savefig('200feats_Correlation.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best_features.sort_values(by=\"importance\", ascending=False).head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_predictions.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CV RMSE score: 3.64582, 3.64718, 3.64517, 3.64480, Best_3.5426\ncreate_file_for_submission(\"lgbm_opt_130KS_merid_puramt_date_histmer_rdiff_w2v_installments_K5.csv\", card_ids, lgbm_predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train.drop(columns=features_to_remove, axis=1, inplace=True)\n# X_test.drop(columns=features_to_remove, axis=1, inplace=True)\n# X_train.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loaded_train_feats = pd.read_csv('../input/test-predictions/X_train_137_KSfeats.csv')\n# loaded_train_feats.shape\n\n# excluded_features, list_discarded\n\n# col_difference = set(loaded_train_feats.columns).symmetric_difference(X[train_features].columns)\n# print(\"Difference in train and test features: {}\".format(len(col_difference)))\n# col_difference\n\n# Difference in train and test features: 6\n# {'card_id', 'first_active_month', 'hist_category_3_purchase_amount_sum_mean',\n# 'hist_installments_mean', 'hist_purchase_amount_max', 'target'}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.to_csv('X_train_137_KSfeats.csv', index=False)\nX_test.to_csv('X_test_136_KSfeats.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\n\nFileLink(r'lgbm_85_feats_catid_median_KFold17.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Progressions :\n> RandomForestRegressor with GridSearchCV using 3 features
\n> Rank : 3719 out of 4110 [Private Score : 3.81301]
\n> Rank : 3749 out of 4110 [Public Score : 3.93004]\n\n> RandomForestRegressor(max_features=2, n_estimators=30) with some historical features
\n> Rank : 3416 out of 4110 [Private Score : 3.79375]
\n> Rank : 3238 out of 4110 [Public Score : 3.87633]\n\n> XGBRegressor with 45 historical and new merchant features with RMSE of 3.7006.
\n> validation_0-rmse:1.78531\tvalidation_1-rmse:3.75632
\n> Rank : 2757 out of 4110 [Private Score : 3.65051] (67.08%)
\n> Rank : 2824 out of 4110 [Public Score : 3.74154] (68.71%)\n\n> XGBRegressor with 59 historical and new merchant features with RMSE of 3.6837.
\n> validation_0-rmse:1.54589\tvalidation_1-rmse:3.73675
\n> Rank : 2755 out of 4110 [Private Score : 3.65021] (67.03%)
\n> Rank : 2818 out of 4110 [Public Score : 3.74044] (68.56%)\n\n> XGBRegressor with 69 historical and new merchant features with RMSE of 3.6837.
\n> validation_0-rmse:1.60083\tvalidation_1-rmse:3.73686
\n> Final RMSE : 3.686220904191716
\n> Rank : 2756 out of 4110 [Private Score : 3.65033] (67.06%)
\n> Rank : 2781 out of 4110 [Public Score : 3.73777] (67.66%)\n\n> XGBRegressor with 79 historical (10 OHE for features1-3) and new merchant features with RMSE of 3.7027.
\n> validation_0-rmse:1.51994\tvalidation_1-rmse:3.73630
\n> Final RMSE : 3.6854681990466274
\n> Rank : 2747 out of 4110 [Private Score : 3.64916] (66.84%)
\n> Rank : 2756 out of 4110 [Public Score : 3.73545] (67.06%)
\n\n> LGBM with 79 historical (10 OHE for features1-3) and new merchant features with RMSE of 3.6545.
\n> [1041] training's rmse: 3.49544\tvalid_1's rmse: 3.60891\n> CV score: 3.65450
\n> Rank : 2115 out of 4110 [Private Score : 3.62122] (51.46%)
\n> Rank : 2370 out of 4110 [Public Score : 3.70550] (57.66%)
\n\n> LGBM with 83 (10 OHE for features1-3 and new merchant purchase date min max mode) features at RMSE of 3.65311.
\n> [1120] training's rmse: 3.4844\tvalid_1's rmse: 3.6065\n> CV score: 3.65311
\n> Rank : 1947 out of 4110 [Private Score : 3.62025] (47.37%)
\n> Rank : 2296 out of 4110 [Public Score : 3.70437] (55.86%)
\n\n> LGBM with 87 using median for nan (new merchant purchase date min max mode with merchant data) features.
\n> [1102] training's rmse: 3.47996\tvalid_1's rmse: 3.60637\n> CV RMSE score: 3.65275
\n> Rank : 1498 out of 4110 [Private Score : 3.61882] (36.45%)
\n> Rank : 2227 out of 4110 [Public Score : 3.70239] (54.18%)
\n\n> LGBM_KFold7 with 95_OHE using median for nan (new merchant purchase date min max mode with merchant data) features.
\n> Rank : 1410 out of 4110 [Private Score : 3.61813] (34.31%)
\n> Rank : 2185 out of 4110 [Public Score : 3.70141] (53.16%)
\n> CV RMSE score: 3.65106\n\n> LGBM_KFold17 with 128 features using median for nan (new merchant purchase date min max mode with merchant data).
\n> Rank : 1805 out of 4110 [Private Score : 3.61959] (43.92%)
\n> Rank : 2184 out of 4110 [Public Score : 3.70135] (53.14%)
\n> CV RMSE score: 3.64977\n\n> LGBM_KFold17 with 136 features using median for nan (new merchant purchase date min max mode with merchant data).
\n> Rank : 1655 out of 4110 [Private Score : 3.61937] (40.27%)
\n> Rank : 2160 out of 4110 [Public Score : 3.70097] (52.55%)
\n> CV RMSE score: 3.64882\n\n> LGBM_KFold17_merchantid with 137 features using median for nan (new merchant purchase date min max mode with merchant data, purchase date max ratio) at CV RMSE of 3.64613.
\n> Rank : 963 out of 4110 [Private Score : 3.61572] (23.43%)
\n> Rank : 1722 out of 4110 [Public Score : 3.69594] (41.90%)
\n> CV RMSE score: 3.64613\n\n> LGBM_KFold17_merchantid with 137 features using median for nan (new merchant purchase date min max mode with merchant data, purchase date max ratio, pd_max) at CV RMSE of 3.64625.
\n> Rank : 922 out of 4110 [Private Score : 3.61546] (22.43%)
\n> Rank : 1672 out of 4110 [Public Score : 3.69539] (40.68%)
\n> CV RMSE score: 3.64625\n> Early stopping, best iteration is:\n> [1152]\ttraining's rmse: 3.47992\tvalid_1's rmse: 3.43485\n\n> LGBM_KFold17_merchantid with 142 features using median for nan (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio') at CV RMSE of 3.64680.
\n> **Rank : 352 out of 4110 [Private Score : 3.61319] (8.56%)**
\n> Rank : 1560 out of 4110 [Public Score : 3.69400] (37.96%)
\n> CV RMSE score: 3.64680\n\n> LGBM_KFold17_merchantid with 138 features using median for nan (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio') at CV RMSE of 3.64599.
\n> Rank : 356 out of 4110 [Private Score : 3.61322] (8.66%)
\n> Rank : 1525 out of 4110 [Public Score : 3.69357] (37.10%)
\n> CV RMSE score: 3.64599
\n> Wall time: 37min 5s\n\n> LGBM_KFold17_merchantid with 139 features using median for nan (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'hist_category_1_purchase_amount_sum_mean).
\n> Rank : 405 out of 4110 [Private Score : 3.61363] (9.85%)
\n> Rank : 1504 out of 4110 [Public Score : 3.69340] (36.59%)
\n> CV RMSE score: 3.64582
\n> Wall time: 36min 25s\n\n> LGBM_KFold17_merchantid with 142 features using median for nan (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'new_hist_purchase_date_min_ratio', 'hist_category_1_purchase_amount_sum_mean', 'new_merchant_category_1_purchase_amount_sum_mean', ratio and diff) at CV RMSE of 3.64659.\n> Rank : 384 out of 4110 [Private Score : 3.61342] (9.34%)
\n> Rank : 1447 out of 4110 [Public Score : 3.69303] (35.21%)
\n> CV RMSE score: 3.64659
\n> Wall time: 37min 26s
\n\n> LGBM_KFold17_merchantid with 142 features using median for nan (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'new_hist_purchase_date_min_ratio', 'hist_category_1_purchase_amount_sum_mean', 'new_merchant_category_1_purchase_amount_sum_mean', ratio and diff) at CV RMSE of 3.64659.\n> Rank : 795 out of 4110 [Private Score : 3.61475] (19.34%)
\n> Rank : 1253 out of 4110 [Public Score : 3.69242] (30.49)
\n> CV RMSE score: 3.64595
\n> Wall time: 36min 42s
\n\n> LGBM_KFold17_merchantid with 150 features (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'new_hist_purchase_date_min_ratio', 'hist_category_1_purchase_amount_sum_mean', 'new_merchant_category_1_purchase_amount_sum_mean', ratio and diff, 'hist_w2v_hist_merchant_cat_id_std').\n> Rank : 687 out of 4110 [Private Score : 3.61394] (16.72%)
\n> Rank : 1236 out of 4110 [Public Score : 3.69227] (30.07%)
\n> CV RMSE score: 3.64573
\n> Wall time: 37min 46s
\n\n> LGBM_KFold17_merchantid with 162 features (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'new_hist_purchase_date_min_ratio', 'hist_category_1_purchase_amount_sum_mean', 'new_merchant_category_1_purchase_amount_sum_mean', 'w2v_hist_merchant_category_id_0-4_std', 'w2v_hist_merchant_id_0-4_std', 'w2v_new_merchant_category_id_0-4_std', 'w2v_new_merchant_id_0-4_std' ratio and diff interactions).
\n> Rank : 955 out of 4110 [Private Score : 3.61569] (23.24%)
\n> Rank : 1066 out of 4110 [Public Score : 3.69122] (25.94%)
\n> CV RMSE score: 3.64517
\n> Wall time: 40min 32s
\n\n> LGBM_KFold17_merchantid with 162 features (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'new_hist_purchase_date_min_ratio', 'hist_category_1_purchase_amount_sum_mean', 'new_merchant_category_1_purchase_amount_sum_mean', 'w2v_hist_merchant_category_id_0-4_std', 'w2v_hist_merchant_id_0-4_std', 'w2v_new_merchant_category_id_0-4_std', 'w2v_new_merchant_id_0-4_std' ratio and diff interactions).
\n> Rank : 908 out of 4110 [Private Score : 3.61537] (22.09%)
\n> Rank : 1020 out of 4110 [Public Score : 3.69069] (24.82%)
\n> CV RMSE score: 3.64634
\n> Wall time: 47min 6s
\n\n> LGBM_KFold17_merchantid with 130 or 133 KS_0.01features (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'new_hist_purchase_date_min_ratio', 'hist_category_1_purchase_amount_sum_mean', 'new_merchant_category_1_purchase_amount_sum_mean', 'w2v_hist_merchant_category_id_0-4_std', 'w2v_hist_merchant_id_0-4_std', 'w2v_new_merchant_category_id_0-4_std', 'w2v_new_merchant_id_0-4_std', ratio and diff interactions, installments_outliers_median).
\n> Rank : 990 out of 4110 [Private Score : 3.61597] (24.09%)
\n> Rank : 920 out of 4110 [Public Score : 3.68906] (22.38%)
\n> CV RMSE score: 3.64537
\n> Wall time: 31min 40s
\n\n> LGBM_KFold17_merchantid_opt with 130 KS_0.01features (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'new_hist_purchase_date_min_ratio', 'hist_category_1_purchase_amount_sum_mean', 'new_merchant_category_1_purchase_amount_sum_mean', 'w2v_hist_merchant_category_id_0-4_std', 'w2v_hist_merchant_id_0-4_std', 'w2v_new_merchant_category_id_0-4_std', 'w2v_new_merchant_id_0-4_std', ratio and diff interactions, installments_outliers_median).
\n> Rank : 948 out of 4110 [Private Score : 3.61562] (23.07%)
\n> Rank : 869 out of 4110 [Public Score : 3.68811] (21.14%)
\n> CV RMSE score: 3.64531
\n> Wall time: 32min 36s
\n\n> LGBM_KFold5_merchantid_opt2 with 130 KS_0.01features (new merchant purchase date min max mode with merchant data, 'new_hist_purchase_date_max_ratio', pd_max, 'new_hist_purchase_amount_max_ratio', 'new_hist_purchase_date_min_ratio', 'hist_category_1_purchase_amount_sum_mean', 'new_merchant_category_1_purchase_amount_sum_mean', 'w2v_hist_merchant_category_id_0-4_std', 'w2v_hist_merchant_id_0-4_std', 'w2v_new_merchant_category_id_0-4_std', 'w2v_new_merchant_id_0-4_std', ratio and diff interactions, installments_outliers_median).
\n> (5 KFold performs better than 17 KFold)
\n> Rank : 1415 out of 4110 [Private Score : 3.61817] (34.43%)
\n> **Rank : 495 out of 4110 [Public Score : 3.68718] (12.04%)**
\n> CV RMSE score: 3.65024
\n> Wall time: 8min 31s
\nEarly stopping, best iteration is:\n[1233]\ttraining's rmse: 3.38217\tvalid_1's rmse: 3.63805\n\n--\nEarly stopping, best iteration is:\n[1164]\ttraining's rmse: 3.41147\tvalid_1's rmse: 3.4898\n--\nEarly stopping, best iteration is:\n[826]\ttraining's rmse: 3.49935\tvalid_1's rmse: 3.49333\n\n--\nEarly stopping, best iteration is:\n[1892]\ttraining's rmse: 3.38284\tvalid_1's rmse: 3.48561\nCV RMSE score: 3.64479 \n--\nEarly stopping, best iteration is:\n[902]\ttraining's rmse: 3.48923\tvalid_1's rmse: 3.48783\nCV RMSE score: 3.64541\n--\n\n--\n> Remove the 'new_merchant_purchase_date_max' and 'hist_purchase_date_max' after creating the ratio\n> Early stopping, best iteration is:\n> [1007]\ttraining's rmse: 3.49581\tvalid_1's rmse: 3.43503\n> CV RMSE score: 3.64655 \n> Wall time: 52min 27s","metadata":{}},{"cell_type":"markdown","source":"\n\n","metadata":{}},{"cell_type":"markdown","source":"# (D) Feature Extractions (with Sequential Feed-forward Neural Network), Deep Learning\n- #### Train our model
\n> For training a model, we use the fit function, which trains the model for a given number of epochs, which refers to the number of times we pass our dataset through our model to train it. We use callbacks.TensorBoard to writes a log for TensorBoard, which allows you to visualize dynamic graphs of your training and test metrics, as well as activation histograms for the different layers in your model. To save our model after every epoch we using callbacks.ModelCheckpoint for it.","metadata":{}},{"cell_type":"markdown","source":"- #### Import libraries for Deep Learning","metadata":{}},{"cell_type":"code","source":"import keras\nfrom keras import callbacks\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, BatchNormalization\nfrom tensorflow.keras import models, layers\n# from keras.optimizers import RMSprop, SGD\nfrom keras.optimizers import Adam, Adadelta\nfrom keras.optimizers.schedules import ExponentialDecay\nfrom keras.metrics import RootMeanSquaredError\n\n# For loading saved model in h5 format\nfrom keras.models import load_model\n\n#As shown:\n#1) No convolution layer.\n#2) Make the input shape equals to your regression features numbers.\n#3) The network ends with output layer with only 1 output and no activation (it will be a linear layer, applying an activation function would constrain the range the output can take)\n#4) Compile the network with the mse loss function—mean squared error, the square of the difference between the predictions and the targets. This is a widely used loss function for regression problems.\n#5) Make the metric to monitor during training as mean absolute error (MAE). It’s the absolute value of the difference between the predictions and the targets. ","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:31:54.560622Z","iopub.execute_input":"2021-11-09T10:31:54.561200Z","iopub.status.idle":"2021-11-09T10:31:59.984016Z","shell.execute_reply.started":"2021-11-09T10:31:54.561148Z","shell.execute_reply":"2021-11-09T10:31:59.982805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tensorboard = callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True)\nmodel_checkpoints = callbacks.ModelCheckpoint(\"weights_{epoch:02d}_{val_loss:.2f}.h5\", monitor='val_loss',\n verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1)","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:32:23.782149Z","iopub.execute_input":"2021-11-09T10:32:23.782514Z","iopub.status.idle":"2021-11-09T10:32:23.787893Z","shell.execute_reply.started":"2021-11-09T10:32:23.782482Z","shell.execute_reply":"2021-11-09T10:32:23.786883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(input_df):\n print(\"No. of features :\", input_df.shape[1])\n model = models.Sequential()\n model.add(layers.Dense(512, activation='relu', input_shape=(input_df.shape[1],))) # 1024\n model.add(layers.Dropout(0.2)),\n model.add(layers.Dense(256, activation='relu')) # 512\n model.add(layers.Dropout(0.3)),\n model.add(layers.Dense(128, activation='relu')) # 128 \n model.add(layers.Dropout(0.5)),\n model.add(layers.Dense(1))\n\n lr_schedule = ExponentialDecay(initial_learning_rate=1e-3,\n decay_steps=10000, decay_rate=0.9)\n \n model.compile(optimizer=Adam(learning_rate=1e-3), # Adadelta(learning_rate=0.001, rho=0.95, epsilon=1e-07, name=\"Adadelta\")\n loss='mean_squared_error', # mse\n metrics=['RootMeanSquaredError'])\n return model","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:32:25.734852Z","iopub.execute_input":"2021-11-09T10:32:25.735542Z","iopub.status.idle":"2021-11-09T10:32:25.744016Z","shell.execute_reply.started":"2021-11-09T10:32:25.735505Z","shell.execute_reply":"2021-11-09T10:32:25.742871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import keras\n# from keras import callbacks\n# from keras.models import Sequential\n# from keras.layers import Dense, Dropout, BatchNormalization\n# from tensorflow.keras import models, layers\n# # from keras.optimizers import RMSprop, SGD\n# from keras.optimizers import Adam, Adadelta\n# from keras.optimizers.schedules import ExponentialDecay\n# from keras.metrics import RootMeanSquaredError\n\n# def build_model(input_df):\n# print(\"No. of features :\", input_df.shape[1])\n# model = models.Sequential()\n\n# model.add(Dense(2 ** 10, input_dim = input_df.shape[1],\n# kernel_initializer='glorot_uniform', activation='relu')) # init='random_uniform',\n# model.add(Dropout(0.25)) \n# model.add(BatchNormalization())\n# model.add(Dense(2 ** 9, activation='relu')) # kernel_initializer='random_uniform'\n# model.add(BatchNormalization())\n# model.add(Dropout(0.25)) \n# model.add(Dense(2 ** 5, activation='relu')) # kernel_initializer='random_uniform'\n# model.add(BatchNormalization())\n# model.add(Dropout(0.25)) \n# model.add(Dense(1))\n\n# # model.compile(loss='mean_squared_error', optimizer='adam')\n# model.compile(optimizer=Adam(learning_rate=1e-3), # Adadelta(learning_rate=0.001, rho=0.95, epsilon=1e-07, name=\"Adadelta\")\n# loss='mse',\n# metrics=['RootMeanSquaredError'])\n# return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[train_features].shape, X_test[train_features].shape","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:32:29.819289Z","iopub.execute_input":"2021-11-09T10:32:29.819801Z","iopub.status.idle":"2021-11-09T10:32:29.942957Z","shell.execute_reply.started":"2021-11-09T10:32:29.819741Z","shell.execute_reply":"2021-11-09T10:32:29.942149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dl_model = build_model(X[train_features])\n\ndl_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:32:30.444898Z","iopub.execute_input":"2021-11-09T10:32:30.445544Z","iopub.status.idle":"2021-11-09T10:32:30.634438Z","shell.execute_reply.started":"2021-11-09T10:32:30.445490Z","shell.execute_reply":"2021-11-09T10:32:30.633575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel_log = dl_model.fit(X[train_features], y, epochs=20, batch_size=64, # 16\n validation_split = 0.2,\n callbacks=[model_checkpoints]) # , verbose=0)\n\n# Beat this, val_root_mean_squared_error: 3.9318","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:32:39.315807Z","iopub.execute_input":"2021-11-09T10:32:39.316410Z","iopub.status.idle":"2021-11-09T10:36:50.455027Z","shell.execute_reply.started":"2021-11-09T10:32:39.316375Z","shell.execute_reply":"2021-11-09T10:36:50.453578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, (ax1, ax2) = plt.subplots(1, 2,figsize=(15,5))\n\nax1.plot(model_log.history['root_mean_squared_error'])\nax1.plot(model_log.history['val_root_mean_squared_error'])\nax1.set_title('Root Mean Squared Error (RMSE)')\nax1.set(xlabel='Epoch', ylabel='RMSE')\nax1.legend(['train', 'validation'], loc='upper right')\nax1.xaxis.set_major_locator(MaxNLocator(integer=True))\n\nax2.plot(model_log.history['loss'])\nax2.plot(model_log.history['val_loss'])\nax2.set_title('Loss (Lower Better)')\nax2.set(xlabel='Epoch', ylabel='Loss')\nax2.legend(['train', 'validation'], loc='upper right')\nax2.xaxis.set_major_locator(MaxNLocator(integer=True))\n\nplt.savefig('130KSfeats_DL3_train_validation.png')","metadata":{"execution":{"iopub.status.busy":"2021-11-09T10:38:40.833062Z","iopub.execute_input":"2021-11-09T10:38:40.833555Z","iopub.status.idle":"2021-11-09T10:38:41.558311Z","shell.execute_reply.started":"2021-11-09T10:38:40.833512Z","shell.execute_reply":"2021-11-09T10:38:41.557151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# first_batchdata = X[train_features].iloc[:5]\n# first_batchlabels = y.iloc[:5]\n\n# print(\"XBGR Predictions: \", xgb_model.predict(first_batchdata))\n# print(\"LGBM Predictions: \", lbg_reg.predict(first_batchdata))\n# print(\"DL Predictions: \", dl_model.predict(first_batchdata))\n\n# print(\"Labels: \", list(first_batchlabels))\n\n# test_mse_score, test_rmse_score = dl_model.evaluate(first_batchdata, first_batchlabels)\n\n# print(f'test_mse_score : {test_mse_score}')\n# print(f'test_rmse_score : {test_rmse_score}')\n\n# # The attribute model.metrics_names will give you the display labels for the evalution outputs.\n# print(\"\\nModel metrics :\", dl_model.metrics_names)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dl_predictions = dl_model.predict(X_test[train_features])\ndl_predictions.shape, X_test[train_features].shape\n\n# DL_138 feats with Pri_score : 3.83928, Pub_score : 3.95739\n# DL_130 KSfeats with Pri_score : 3.84114, Pub_score : 3.95931\n# DL3_v1_130 KSfeats with Pri_score : 3.82566, Pub_score : 3.94311","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://keras.io/api/models/model_saving_apis/\ndl_model.save('DL3_v1_130KSfeat_adam.h5')\n\n# deletes the existing model\n# del dl_model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_file_for_submission(\"DL3_v1_130KSfeat_adam.csv\", card_ids, dl_predictions[:, 0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (E) Blending prediction results from different models (heterogeneous regressors) by averaging","metadata":{}},{"cell_type":"code","source":"!ls -tl ../input/test-predictions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ## Heterogeneous regressors predictions","metadata":{}},{"cell_type":"code","source":"# Load from saved predictions from different models\n# '../input/testpredictions/lgbm_162_feats_KFold17.csv'\nlgbm_1_pred = pd.read_csv('lgbm_opt2_130KS_merid_puramt_date_histmer_rdiff_w2v_installments_K5.csv')\nlgbm_2_pred = pd.read_csv('../input/test-predictions/lgbm_130KSfeats_K17.csv')\nlgbm_3_pred = pd.read_csv('../input/test-predictions/lgbm_141_feats.csv')\nxgbr_1_pred = pd.read_csv('../input/test-predictions/xgbr_126KS_feats_KFold17.csv')\nxgbr_2_pred = pd.read_csv('../input/test-predictions/xgbr_141_feats.csv')\nrf_pred = pd.read_csv('../input/test-predictions/rf_130KS_feats.csv')\ndl_pred = pd.read_csv('DL3_v1_130KSfeat_adam.csv') # dl3_130KS_feats.csv'\n\nconcat_test_pred = pd.concat([lgbm_1_pred['target'], lgbm_2_pred['target'], lgbm_3_pred['target'],\n xgbr_1_pred['target'], xgbr_2_pred['target'],\n rf_pred['target'], dl_pred['target']],\n axis=1, keys=['LGBM_1_Pred', 'LGBM_2_Pred', 'LGBM_3_Pred',\n 'XGBR_1_Pred', 'XGBR_2_Pred',\n 'RF_Pred', 'DL_Pred'])\n\n# dl_preds = pd.read_csv('../input/testpredictions/xgbr_lgbm_dl_79feat.csv')\n# concat_pred_df = pd.concat([xgbr_preds['target'], lgbm_preds['target'], dl_preds['target']],\n# axis=1, keys=['XGBR_Pred', 'LGBM_Pred', 'DL_Pred'])\n\n# concat_pred_df['Avg_Pred'] = concat_pred_df.mean(axis=1)\n\n# xgbr_preds.loc[:5, 'target'], lgbm_preds.loc[:5, 'target'], dl_preds.loc[:5, 'target']\n\nconcat_test_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgbr_predictions_df = pd.DataFrame(xgbr_predictions, columns = ['XGBR_Pred'])\n# lgbm_predictions_df = pd.DataFrame(lgbm_predictions, columns = ['LGBM_Pred'])\n# dtreg_predictions_df = pd.DataFrame(tree_reg_predictions, columns = ['DTR_Pred'])\n# rfb_predictions_df = pd.DataFrame(rf_predictions, columns = ['RFB_Pred'])\n# dl_predictions_df = pd.DataFrame(dl_predictions[:, 0], columns = ['DL_Pred'])\n\n# concat_pred_df = pd.concat([xgbr_predictions_df, lgbm_predictions_df, dtreg_predictions_df], axis=1, sort=False)\n# concat_pred_df = pd.concat([concat_pred_df, rfb_predictions_df], axis=1, sort=False)\n# concat_pred_df = pd.concat([concat_pred_df, dl_predictions_df], axis=1, sort=False)\n\n# concat_pred_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concat_test_pred['Avg_Pred'] = concat_test_pred.mean(axis=1)\nconcat_test_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[train_features].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Average predicted values from XGBR, LGBM, RF and DL\ncreate_file_for_submission(\"lgbm_xgbr_rfb_dl3_130feat.csv\", card_ids, concat_test_pred['Avg_Pred'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# (F) Postprocessing","metadata":{}},{"cell_type":"code","source":"predicted_loyality_score = concat_test_pred['LGBM_1_Pred']\nax = predicted_loyality_score.plot.hist(bins=20, figsize=(6, 5))\n_ = ax.set_title(\"Predicted target histogram\")\nplt.show()\n\nfig, axs = plt.subplots(1,2, figsize=(12, 5))\n_ = predicted_loyality_score[predicted_loyality_score > 10].plot.hist(ax=axs[0])\n_ = axs[0].set_title(\"Predicted target histogram for values greater than 10\")\n_ = predicted_loyality_score[predicted_loyality_score < -10].plot.hist(ax=axs[1])\n_ = axs[1].set_title(\"Predicted target histogram for values less than -10\")\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_pred = lgbm_1_pred\nlgbm_pred.loc[lgbm_pred['target'] < -10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predicted 'target' less than -10 is considered outliers\ncard_id_lt_neg10 = lgbm_pred.loc[lgbm_pred['target'] < -10, 'card_id']\nlen(list(card_id_lt_neg10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Post-processing by assigning cards with outlier 'target' value\nfor id in card_id_lt_neg10:\n lgbm_pred.loc[lgbm_pred['card_id']==id,'target'] = -33.21928095\n \nlgbm_pred.loc[lgbm_pred['target'] < -10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Average predicted values from LGBM, XGBR, RF and DL\ncreate_file_for_submission(\"lgbm_130KSfeats_postprocess_.csv\", card_ids, lgbm_pred['target'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* # Conclusions :\n> #### 1. Single LGBM with KS 130 features performs the best\n> #### 2. Outlier predictions are important for this problem; thus the multiple model combinations like blending method (3) performs well.\n> #### 3. With hyperparameter tunings and post processing can potentially improve on the ML models.\n> #### 4. More feature engineering with KS could also improve the final model for the problem.","metadata":{}},{"cell_type":"markdown","source":"# Part 4 : Visualizations","metadata":{}},{"cell_type":"code","source":"concat_test_pred.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- #### Comparisons on prediction results from different models","metadata":{}},{"cell_type":"code","source":"no_of_samples = 50\nlgbm_pred_samples = concat_test_pred.loc[:no_of_samples, 'LGBM_1_Pred']\nxgbr_pred_samples = concat_test_pred.loc[:no_of_samples, 'XGBR_1_Pred']\nrf_pred_samples = concat_test_pred.loc[:no_of_samples, 'RF_Pred']\ndl_pred_samples = concat_test_pred.loc[:no_of_samples, 'DL_Pred']\n\n# rfb_pred_samples = concat_pred_df.loc[:no_of_samples, 'RFB_Pred']\n#dtreg_pred_samples = concat_pred_df.loc[:no_of_samples, 'DTR_Pred']\n\nmodel_avg_pred_samples = concat_test_pred.loc[:no_of_samples, 'Avg_Pred']\n# esm_reg_pred_samples = ensemble_reg_pred.loc[:no_of_samples, 'ESM_LGBM_Pred']\n\n# https://matplotlib.org/stable/tutorials/colors/colors.html\n# https://matplotlib.org/stable/api/markers_api.html\n\nplt.figure(figsize=(20,12))\nplt.plot(lgbm_pred_samples, 'b^', label='LGBM_1')\nplt.plot(xgbr_pred_samples, 'gd', label='XGBR_1')\nplt.plot(rf_pred_samples, 'ys', label='RFB')\nplt.plot(dl_pred_samples, 'c+', label='DL3') # DL5\nplt.plot(model_avg_pred_samples, 'r*', ms=10, label='Avg')\n\n# plt.plot(esm_reg_pred_samples, 'm.', ms=10, label='ESM_LGBM')\n\nplt.tick_params(axis='x', which='both', bottom=False, top=False,\n labelbottom=False)\n\nplt.ylabel('predicted')\nplt.xlabel('Test samples')\nplt.legend(loc=\"best\")\nplt.title('Model predictions with their averages and stacking')\n\n\nplt.savefig('Model_Prediction_plots.png')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## End of Program","metadata":{}}]}