{"cells":[{"metadata":{"_uuid":"ad57fd08a788eea2837e7c7180291618237e01b7"},"cell_type":"markdown","source":"![](https://d1yjjnpx0p53s8.cloudfront.net/styles/logo-thumbnail/s3/022016/logo_elo.jpg?itok=1XSbdYjp)"},{"metadata":{"_uuid":"1f1dd2a438e24824a38a96e1a1bb97fc3d208848"},"cell_type":"markdown","source":"**About Elo**\n\n[Elo](https://www.cartaoelo.com.br/) is largest domestic credit/debit card company in Brazil and processes payments in local currency. It has issued more than 50k cards. Elo cards allow shoppers to spread the purchase payments over certain period and can be paid in installments. \n\n**Objective**\nData of historical transcation for each card user and merchant is provided. Objective of this competition is to predict the loyalty score for each card user. "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4660a4b53b7b37ae67bb7f50d7f356f83f8cd88c"},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\ndef missing_median(df, name):\n    med = df[name].median()\n    df[name] = df[name].fillna(med)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# This contains training data with features and targe variable. Names are masked. \ndata_train = pd.read_csv('../input/train.csv',parse_dates=['first_active_month'])\n\n# This file contains test data \ndata_test = pd.read_csv('../input/test.csv',parse_dates=['first_active_month'])\n\n# This file contains additional information about all merchants / merchant_ids in the dataset\n#merchants = pd.read_csv('../input/merchants.csv')\nnew_merch_trans = pd.read_csv('../input/new_merchant_transactions.csv',parse_dates =['purchase_date'])\n\n# This file contains up to 3 months' worth of historical transactions for each card_id\nhist_trans = pd.read_csv('../input/historical_transactions.csv',parse_dates =['purchase_date'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a660a59bb1c7aabe7cd3789032438cd1afd89e0e"},"cell_type":"code","source":"print('Null data in training data')\nprint(data_train.isnull().sum())\nprint('Null data in test data')\nprint(data_test.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bce140a0e2c9e656504dd28b437de8fa4e6f6bc"},"cell_type":"code","source":"dummydate = data_test['first_active_month'][0]\ndata_test['first_active_month'].fillna(dummydate,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd450e141dc2b26eb389186b132a31a7eebe5fa5"},"cell_type":"code","source":"print (dummydate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9bd3effda3947ea4c66d71547dc454320246c19"},"cell_type":"code","source":"# Check what is inside of these files first. \ndata_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1ac1efa4446ae54643a0baa743ace5d4d16e0e6"},"cell_type":"code","source":"#data_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fdb62df9a11facebf813a71c8e9c42e5b68e121"},"cell_type":"code","source":"data_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53070ca6ed3d577db9dee99a7e0fba26e18f8721"},"cell_type":"code","source":"plt.scatter(range(data_train.shape[0]), np.sort(data_train['target'].values))\nplt.xlabel('index', fontsize=12)\nplt.ylabel('Loyalty Score', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a54cc3f8dcdeb964180032496a634e7d79cdfdf"},"cell_type":"markdown","source":"Above variation in loyalty score indicates loyalty score increase gradually first and then exponentially. Outliers on both side matters for the prediction of the model. "},{"metadata":{"trusted":true,"_uuid":"83d562eb61427d5c0a2579c5d1dad22193e3d7c1"},"cell_type":"code","source":"sns.set()\ndata_train['target'].plot(kind='hist')\nplt.xlabel('Target')\nplt.title('Variation of Target Values')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01d5206530bee12ebacfd9d9521a002969806c3b"},"cell_type":"code","source":"# This file contains additional information about all merchants / merchant_ids in the dataset.\n# Important features here are category_id, merchant_group_id, subsector_id, city_id, state_id\n#merchants.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8797152fbfee0052cbced7e700191b99bac14dea"},"cell_type":"code","source":"# This file contains up to 3 months' worth of historical transactions for each card_id\n# So this builds some history for every card user at set number of merchants. \nhist_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4ed029c1d4f39a7d236373e6ddf1b1f77d1a362"},"cell_type":"code","source":"# This file contains two months worth of data for each card_id containing ALL purchases that card_id made at merchant_ids \n# that were not visited in the historical data.\n\n# So apart from historical data, new history checks if the card user is spending more on new categories\n\nnew_merch_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8311f94e7d41b1fec48e483a0aef57d47fffd708"},"cell_type":"markdown","source":"**Data Preparation**\n\n* Split the date and time in data_train and data_test\n* Map Y/N to 1-0 in authorised and catetogry.\n*  Categorical data - Category_3, Category_2\n*  Drop exterem loyalty score values in the training data less than -10"},{"metadata":{"trusted":true,"_uuid":"f0140ff390c2eaac0b700953aa4bc6598d7bf10a"},"cell_type":"code","source":"# Map the columns with helping function from \n# Ref: https://www.kaggle.com/fabiendaniel/elo-world\ndef binarize(df):\n    for column in ['authorized_flag', 'category_1']:\n        df[column] = df[column].map({'Y':1, 'N':0})\n    return df\n\nhist_trans_prep = binarize(hist_trans)\nnew_merch_trans_prep = binarize(new_merch_trans)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43cc465fa49907838a7b12276ce18dd8c2204cf6"},"cell_type":"code","source":"hist_trans_prep = reduce_mem_usage(hist_trans_prep)\nnew_merch_trans_prep = reduce_mem_usage(new_merch_trans_prep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78c8b2e08a3e9ba181e4d726b6d283ada5b3612b"},"cell_type":"code","source":"def split_time2(df):\n    df['pd_year'] = df.purchase_date.dt.year\n    df['pd_month'] = df.purchase_date.dt.month\n    df['pd_day_of_year'] = df.purchase_date.dt.dayofyear\n    df['pd_day_of_week'] = df.purchase_date.dt.dayofweek\n    df['pd_hour'] = df.purchase_date.dt.hour\n    return df\n\nhist_trans_prep = split_time2(hist_trans_prep)\nnew_merch_trans_prep = split_time2(new_merch_trans_prep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46f7b69b0330a57e8d074f502d8764a2a54fffe0"},"cell_type":"code","source":"# Drop categorical and date columns\nhist_trans_prep.drop('purchase_date',1,inplace=True)\nnew_merch_trans_prep.drop('purchase_date',1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c9dbbc9732f7fa93d60d5b3484f6fda3fd14505"},"cell_type":"code","source":"# Categorical data dummy creation for transaction data\n\n# Create categorical columns\nfeatures2 = ['category_2', 'category_3']\n\n# get dummies\nhist_trans_prep = pd.get_dummies(hist_trans_prep,columns=features2)\nnew_merch_trans_prep = pd.get_dummies(new_merch_trans_prep,columns=features2)\nhist_trans_prep = reduce_mem_usage(hist_trans_prep)\nnew_merch_trans_prep = reduce_mem_usage(new_merch_trans_prep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfd2f3c6186eba5f88539a40591c0b459aef635b"},"cell_type":"code","source":"def split_time(df):\n    df['fac_year'] = df.first_active_month.dt.year\n    df['fac_month'] = df.first_active_month.dt.month\n    df['fac_day_of_year'] = df.first_active_month.dt.dayofyear\n    df['fac_day_of_week'] = df.first_active_month.dt.dayofweek\n    return df\n    \ndata_train_prep = split_time(data_train)\ndata_test_prep = split_time(data_test)\ndata_train_prep.drop('first_active_month',1,inplace=True)\ndata_test_prep.drop('first_active_month',1,inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f523862893776ec521224870a9c533befb8c14f5"},"cell_type":"code","source":"data_train_prep.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"995485f78f63ec8417d106fd353381d4434b8458"},"cell_type":"code","source":"#data_train_prep = data_train_prep[data_train_prep['target']>-20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45f1e4327c2f9dcc89213ef2fbe7188d3ba7e6f2"},"cell_type":"code","source":"#final_size = data_train_prep.shape[0]\n#original_size= data_train.shape[0]\n#print('Percentage data dropped after preparation',(original_size-final_size)/original_size*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d124e27c260b767f2b3816d1c9de6edb5ef99513"},"cell_type":"code","source":"# Create categorical columns\nfeatures = ['feature_1','feature_2','feature_3']\n\n# get dummies\ndata_train_prep = pd.get_dummies(data_train_prep,columns=features)\ndata_test_prep = pd.get_dummies(data_test_prep,columns=features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6a9345789262ff00a0e4877262d53812dc67a16"},"cell_type":"code","source":"data_train_prep.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1488aaa32a1393bffb19e0a9bb212ab73a0043fd"},"cell_type":"code","source":"data_test_prep.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34ea9bc2eb581e7255cda1f3fb1c937b01790bea"},"cell_type":"code","source":"hist_trans_prep.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8aa23e4ccde3c0be15b90b48cd570363b1a9cd6"},"cell_type":"code","source":"#hist_trans_prep.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c700ce96d7a2aa8231a95a99221471ca689581d0"},"cell_type":"code","source":"new_merch_trans_prep.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c0625b47ab70b398c0f123a35cf846a28e40fea"},"cell_type":"code","source":"#print('Null data in historical transaction data')\n#print(hist_trans.isnull().sum())\n#print('Null data in new data')\n#print(new_merch_trans.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"60c6161913d957ca10a3edbe8f2fe97bbeff8746"},"cell_type":"markdown","source":"**Notes on establishing model**\n1. Link historical information of merchants visits and purchases to a loyalty score, which is basically a business metric that considered both future spending and retention as main components. \n2. Find a link which triggers higher loyalty scores and you can use any kind of modeling, including recommendation system\n\n**Variables of Higher Influence**\n* Number of authorised transactions- higher the number of authorized transactions, higher the score. % of approval transactions\n* Average purchase amount per purchase in category 2 (1-5) and 3 (A-C) - higher the better score. In data this value is normalized. \n* Number of installments paid  - larger the number of installments the better score, due to higher intrest earned by Elo\n* Month lag - more lag less loyalty score\n* Age of the card - More age means more score, first active date is good indicator. Difference between first active date and use date. Averge it out for all transaction. \n* Number of unique merchant visits - more types of shopping means more business, higher the loyalty score\n* Total purchase - higher the better score\n\n**Additional Features From New Merchants Data** \n* Number of new merchant visits - Total of unique merchants visits\n\nLoyalty numerical score calculated 2 months after evaluation of new merchant data. "},{"metadata":{"trusted":true,"_uuid":"bbe4548afb588c681f3bbc8c796451a093feca4c"},"cell_type":"code","source":"agg_fun = {'authorized_flag': ['sum', 'mean']}\nhist_auth_mean = hist_trans_prep.groupby(['card_id']).agg(agg_fun)\nhist_auth_mean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bcb67a2704e70c6003ec5bf7d3d0560375e8218"},"cell_type":"code","source":"hist_auth_mean.columns = ['_'.join(col).strip() for col in hist_auth_mean.columns.values]\nhist_auth_mean.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea8a833d2d21a909699d9d5178fe59e3d22b6285"},"cell_type":"code","source":"# Sum is total of authorized transactions and mean is % of total transcations\nhist_auth_mean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f56f96228b06897f82488fdaa22118f59df4b3ce"},"cell_type":"code","source":"# Filtering out authorized transactions only for historical data\nhist_trans_prep_auth = hist_trans_prep[hist_trans_prep['authorized_flag']==1]\nhist_trans_prep_auth.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71385c6ea8d684bb4a38987bd1183a51a9ef2844"},"cell_type":"code","source":"def aggregate_transactions(history):\n    agg_func = {\n        'category_1': ['sum', 'mean'],\n        'category_2_1.0': ['mean'],\n        'category_2_2.0': ['mean'],\n        'category_2_3.0': ['mean'],\n        'category_2_4.0': ['mean'],\n        'category_2_5.0': ['mean'],\n        'category_3_A': ['mean'],\n        'category_3_B': ['mean'],\n        'category_3_C': ['mean'],\n        'merchant_id': ['nunique'],\n        'merchant_category_id': ['nunique'],\n        'state_id': ['nunique'],\n        'city_id': ['nunique'],\n        'subsector_id': ['nunique'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min'],\n        'installments': ['sum', 'mean', 'max', 'min'],\n        'pd_month': ['mean', 'max', 'min'],\n        'pd_year': [np.ptp, 'min', 'max'],\n        'month_lag': ['min', 'max']\n        }\n    \n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['_'.join(col).strip() for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    # Get total transaction for each card and then add it as a new column\n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0c71dc6b134fdd5d718a276fa3b7f388bc4b2c4"},"cell_type":"code","source":"history_prep_auth_agg = aggregate_transactions(hist_trans_prep_auth)\nhistory_prep_auth_agg.columns = ['hist_' + c if c != 'card_id' else c for c in history_prep_auth_agg.columns]\nhistory_prep_auth_agg[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"588dbce0d7334bb6ea75a3518cb3152bc01daaf9"},"cell_type":"code","source":"new_merch_prep_agg = aggregate_transactions(new_merch_trans_prep)\nnew_merch_prep_agg.columns = ['new_' + c if c != 'card_id' else c for c in new_merch_prep_agg.columns]\nnew_merch_prep_agg[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90c6423fab0efb66905ddb4595bad4403c25a03b"},"cell_type":"code","source":"data_train_prep_agg = pd.merge(data_train_prep, history_prep_auth_agg, on='card_id', how='left')\ndata_test_prep_agg = pd.merge(data_test_prep, history_prep_auth_agg, on='card_id', how='left')\n\ndata_train_prep_agg = pd.merge(data_train_prep_agg, new_merch_prep_agg, on='card_id', how='left')\ndata_test_prep_agg = pd.merge(data_test_prep_agg, new_merch_prep_agg, on='card_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b3b6dd5b3943a6dc59384149e4b3fba90392e64"},"cell_type":"code","source":"data_train_final = pd.merge(data_train_prep_agg, hist_auth_mean, on='card_id', how='left')\ndata_test_final = pd.merge(data_test_prep_agg, hist_auth_mean, on='card_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c9d7aaa48202e93ec141267c66e411054f45a1c","scrolled":true},"cell_type":"code","source":"data_train_final.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"968e84129fdc4c73d6bf27be77fb0017a5400147"},"cell_type":"code","source":"data_train_final.insert(1, 'age_months', ((data_train_final['hist_pd_year_max']-data_train_final['fac_year'])*12+ (data_train_final['hist_pd_month_max']-data_train_final['fac_month'])).astype(int))\ndata_test_final.insert(1, 'age_months', ((data_test_final['hist_pd_year_max']-data_test_final['fac_year'])*12+ (data_test_final['hist_pd_month_max']-data_test_final['fac_month'])).astype(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"104d0e46be43387e7b99f22b10fa36473703f5ad"},"cell_type":"code","source":"data_train_final.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c7b3f710911ac18056895199b9cd5769a88b869"},"cell_type":"code","source":"data_test_final.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dfe2b5ed58e4aec6834cccaec7a24c1f8f97dd6"},"cell_type":"code","source":"y_train = data_train_final['target']\n\ncard_ids_train = data_train_final['card_id']\ncard_ids_test = data_test_final['card_id']\n\nX_train = data_train_final.drop(['target','card_id'],axis=1)\nX_test = data_test_final.drop(['card_id'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e0b89fa6c392c5269e43eace622e1e24b1cbc91"},"cell_type":"code","source":"#from sklearn.preprocessing import MinMaxScaler\n\n# Initialize a scaler, then apply it to the features\n#scaler = MinMaxScaler() # default=(0, 1)\n#numerical = ['age_months', 'fac_year', 'fac_month', 'fac_day_of_year', 'fac_day_of_week']\n#num_features = [c for c in X_train.columns if not ('feature_' in c or 'ptp' in c)]\n#print (num_features)\n#print (len(num_features))\n\n#X_train[numerical] = scaler.fit_transform(X_train[numerical])\n#X_test[numerical] = scaler.fit_transform(X_test[numerical])\n#X_train = reduce_mem_usage(X_train)\n#X_test = reduce_mem_usage(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04900f747402d11bc22db9b615854c5359556059"},"cell_type":"code","source":"card_ids_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b297ac05d4c1562bd229e9c26fd640e1e259fec4"},"cell_type":"code","source":"#print('Null data in historical transaction data')\n#print(X_test.isnull().sum())\n#print('Null data in new data')\n#print(new_merch_trans.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"877423be2a8a8c660c7486a58ea5c4f28b2c927e"},"cell_type":"code","source":"# Baseline Model with Linear Regression\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.linear_model import LinearRegression\nfrom math import sqrt\n\n#Reg_model = LinearRegression()\n#Reg_model.fit(X_train, y_train)\n\n\n#y_pred = Reg_model.predict(X_test)\n# With linear model score is 3.86","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30bf4ee18603c09a47f325d32d5f6417b39a6adb"},"cell_type":"code","source":"import lightgbm as lgb\n\nparam = {'num_leaves': 120,\n         'min_data_in_leaf': 148, \n         'objective':'regression',\n         \"metric\" : \"rmse\",\n         'max_depth': 9,\n         'learning_rate': 0.005,\n         \"min_child_samples\": 24,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.7202,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.8125 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n        # \"lambda_l1\": 0.3468,\n         \"random_state\": 4590,\n         \"verbosity\": 1}\n\nparam2 = {'num_leaves': 400,\n         'min_data_in_leaf': 148, \n         'objective':'regression',\n         \"metric\" : \"rmse\",\n         'max_depth': 9,\n         'learning_rate': 0.01,\n         \"min_child_samples\": 24,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.1,\n         \"nthread\": 4,\n         \"random_state\": 4590,\n         \"verbosity\": 1}\n\nfeatures = [c for c in X_train.columns if c not in ['card_id']]\n\n# List of categorical features starting with feature_\ncateg_feats = [c for c in features if 'feature_' in c]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a86cccb20c44bcf243d5136df013caed9b87072"},"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import RepeatedKFold\n\nimport time\n\n\nfolds = KFold(n_splits=5, shuffle=True, random_state=4520)\n#folds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=4520)\n\noof = np.zeros(len(X_train))\npredictions = np.zeros(len(X_test))\nstart = time.time()\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X_train.values, y_train.values)):\n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(X_train.iloc[trn_idx][features],\n                           label=y_train.iloc[trn_idx],\n                           categorical_feature=categ_feats)\n    val_data = lgb.Dataset(X_train.iloc[val_idx][features],\n                           label=y_train.iloc[val_idx],\n                           categorical_feature=categ_feats)\n\n    num_round = 10000\n    clf = lgb.train(param,\n                    trn_data,\n                    num_round,\n                    valid_sets = [trn_data, val_data],\n                    verbose_eval=100,\n                    early_stopping_rounds = 200)\n    \n    oof[val_idx] = clf.predict(X_train.iloc[val_idx][features], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(X_test[features], num_iteration=clf.best_iteration) / folds.n_splits\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof, y_train)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4ca4da5ab8876155353a932b86ba4817748d954"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":card_ids_test.values})\nsub_df['target'] = predictions\nsub_df.to_csv(\"submit_lgbm.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f755334342a00a9ca0defc6f7c455232cc3d6fd"},"cell_type":"markdown","source":"XGBoost Model"},{"metadata":{"trusted":true,"_uuid":"3b2dd2ec003c56c3f3f8332ebde792e992785330"},"cell_type":"code","source":"import xgboost as xgb\n\nxgb_params = {'eta': 0.005, \n              'max_depth': 5, \n              'subsample': 0.8, \n              'colsample_bytree': 0.8, \n              'objective': 'reg:linear', \n              'eval_metric': 'rmse',\n              'verbosity':0\n              }\n\nFOLDs = KFold(n_splits=5, shuffle=True, random_state=42)\n\noof_xgb = np.zeros(len(X_train))\npredictions_xgb = np.zeros(len(X_test))\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(X_train, y_train.values)):\n    trn_data = xgb.DMatrix(data=X_train.iloc[trn_idx], label=y_train.iloc[trn_idx])\n    val_data = xgb.DMatrix(data=X_train.iloc[val_idx], label=y_train.iloc[val_idx])\n    watchlist = [(trn_data, 'train'), (val_data, 'valid')]\n    print(\"xgb \" + str(fold_) + \"-\" * 50)\n    num_round = 500\n    xgb_model = xgb.train(xgb_params, trn_data, num_round, watchlist, early_stopping_rounds=50, verbose_eval=100)\n    oof_xgb[val_idx] = xgb_model.predict(xgb.DMatrix(X_train.iloc[val_idx]), ntree_limit=xgb_model.best_ntree_limit+50)\n    predictions_xgb += xgb_model.predict(xgb.DMatrix(X_test), ntree_limit=xgb_model.best_ntree_limit+50) / FOLDs.n_splits\n    \nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof_xgb, y_train)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d822796826b363e401e9fd812b8fb3edae11c5b"},"cell_type":"code","source":"sub_df2 = pd.DataFrame({\"card_id\":card_ids_test.values})\nsub_df2['target'] = 0.5*predictions+0.5*predictions_xgb\nsub_df2.to_csv(\"submit_ens.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c53fea08c32b80b776e703664072fe8c004527d4"},"cell_type":"code","source":"results_ens = pd.read_csv('submit_ens.csv')\nresults_lgbm = pd.read_csv('submit_lgbm.csv')\n#results_ens.head()\nsns.set()\nresults_ens['target'].plot(kind='hist',bins=10)\nresults_lgbm['target'].plot(kind='hist',alpha=0.5,bins=10)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"50cb602b567af40dec93a10a922cb2855b479af7"},"cell_type":"markdown","source":"**Things to do further**\n\n* Normalize numerical data\n* Take log of target variable - plot and use, antilog of output\n"},{"metadata":{"trusted":true,"_uuid":"7d5e80f08fb6e52fd5c44b8222f3b2aade09a387"},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}