{"cells":[{"metadata":{"_uuid":"7f11c40a17f6fb09592421063cafbd683e6224b0"},"cell_type":"markdown","source":"Loading all libraries and reading files. We will also parse dates while reading through files."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport datetime\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom plotnine import *\nimport matplotlib.pyplot as plt\nimport plotnine\nimport lightgbm as lgb\nfrom mizani.breaks import date_breaks\n\nprint(os.listdir(\"../input\"))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', parse_dates =['first_active_month'])\ntest = pd.read_csv('../input/test.csv', parse_dates =['first_active_month'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16c6304da5d6fafb1644197c7059dca5af69c774"},"cell_type":"code","source":"hist_trans = pd.read_csv('../input/historical_transactions.csv', parse_dates=['purchase_date'])\nnew_trans = pd.read_csv('../input/new_merchant_transactions.csv', parse_dates=['purchase_date'])\nmerchants = pd.read_csv('../input/merchants.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0910c429defe913e90a3f12722fc3a3c5a1c6ca2"},"cell_type":"markdown","source":"Let's have a look at shape"},{"metadata":{"trusted":true,"_uuid":"f1c94341869ea0a07500865bf215b36af3d20dbe"},"cell_type":"code","source":"dfs = [train, test, hist_trans, new_trans, merchants]\nfor d in dfs:\n    print(d.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a907b876693a653c7dc5e11577bb3a7d4e7d9bd5"},"cell_type":"markdown","source":"Also I would like to explore train and test set with describe command. Describe is my favorite command and it gives really nice glimpse of data. "},{"metadata":{"trusted":true,"_uuid":"134a3f2a60f6432fb6eae3524425b2b40c97fd1d"},"cell_type":"code","source":"train.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40e2542a6f2faa53185a3f28b453acd3a1267b5b"},"cell_type":"code","source":"test.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5be17e4794362c65bca9fc2127e1d1dbe2f6c0b2"},"cell_type":"markdown","source":"It looks like there is one null value for first activation column for test dataset. Let's fill it with most common date"},{"metadata":{"trusted":true,"_uuid":"b7c5bcbe98b74d93a4daa64f3b9af8b77c3fe891"},"cell_type":"code","source":"test['first_active_month'] = pd.to_datetime(test['first_active_month'].fillna('2017-09-01 00:00:00'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af1f9ab209e8642615580ed48647cdb36a562cf4"},"cell_type":"markdown","source":"Let's also look at target value distribution in test dataset"},{"metadata":{"trusted":true,"_uuid":"48221f28e3df4ffaa852f222faadaf6555b7146e"},"cell_type":"code","source":"plotnine.options.figure_size = (14, 6)\nggplot(train, aes(x='target')) +\\\ngeom_histogram(bins=30, fill='blue', color='black') +\\\nscale_x_continuous(breaks=range(-40, 20, 10))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17ceffaf0a9c9fde057363144368ff959eda8805"},"cell_type":"markdown","source":"Most of the values belong to range between -10 and 10.\n\nIt is also good to know whether age of card dependancy on target. For this we need to create new featuer by substracting current date from activation date. We will also explore first active month feature with simple bar chart."},{"metadata":{"trusted":true,"_uuid":"467fde75c7d2137d3ce8b4548873b2354be5b990"},"cell_type":"code","source":"train['dayDiffActiveMonth'] = (train['first_active_month'] - pd.to_datetime(datetime.datetime.now().date())).dt.days\ntest['dayDiffActiveMonth'] = (test['first_active_month'] - pd.to_datetime(datetime.datetime.now().date())).dt.days","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"176afc6c5efdaa3395d8d1524ffaafbf5c905429"},"cell_type":"code","source":"plotnine.options.figure_size = (14, 6)\nggplot(train, aes(x='first_active_month')) +\\\ngeom_bar(fill='red', alpha=0.5, color='black') +\\\nscale_x_datetime(breaks=date_breaks('1 month')) +\\\ntheme(axis_text_x=element_text(rotation=90, hjust=1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f511933a4d36029b2103afadb5c86309df445b9a"},"cell_type":"markdown","source":"Now let's look at pearson co-relation between variables."},{"metadata":{"trusted":true,"_uuid":"b46d89bc7a488b0fb34b44ab606cd5f2949ef581"},"cell_type":"code","source":"train.corr()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"333e1fef8b3d39150656e26e1ce685fbfb6b20ba"},"cell_type":"markdown","source":"This shows our manufactured feature - \"dayDiffActiveMonth\" is having highest co-relation with target variable. However there are no strong co-relations. and we need to manufacture other features to make our model strong. So let's explore how we can use historical and new merchant transactions to create new features."},{"metadata":{"trusted":true,"_uuid":"d81b8fea901132d2f2f525442c40d175c09e170e"},"cell_type":"code","source":"hist_trans.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f8ed8adb61eb2c19f24c5ed53dcabc7f9dd428b"},"cell_type":"code","source":"new_trans.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea402509966751de0865bbc27768abdaf326cad9"},"cell_type":"markdown","source":"Both historical and new merchant data got similar structure. In first phase we will just count number of transactions\nand average of amount. I just want to include numeric features into consideration for now for simplicity. I will add\ncategorical feature's details in later part of kernel(if needed)\n\nFollowing features seem numeric :  installments, month_lag, purchase_amount \n\nI would also like to convert to flags into number so we can consider mean of those binary features."},{"metadata":{"trusted":true,"_uuid":"2b78ab2fddfe67552b235b7f3eed8bebb5d979e0"},"cell_type":"code","source":"hist_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b1181b5a58c6877e2c64262bcf709d2ee63c440"},"cell_type":"code","source":"hist_trans['authorized_flag'] = hist_trans['authorized_flag'].map({\"Y\" : 0, \"N\" : 1})\nnew_trans['authorized_flag'] = new_trans['authorized_flag'].map({\"Y\" : 0, \"N\" : 1})\n\nhist_trans['category_1'] = hist_trans['category_1'].map({\"Y\" : 0, \"N\" : 1})\nnew_trans['category_1'] = new_trans['category_1'].map({\"Y\" : 0, \"N\" : 1})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6aa015af5d7154bf77a4617d4ff6d718ba5a9f9"},"cell_type":"code","source":"for c in ['authorized_flag', 'category_1','installments','month_lag','purchase_amount']:\n    hist_transt_amt_summ = hist_trans.groupby(['card_id']).agg({c : ['count', 'sum', 'mean', 'max', 'std']}).reset_index()\n    hist_transt_amt_summ.columns = ['card_id', 'hist_'+ c +'_cnt', 'hist_'+ c +'_sum', \n                                    'hist_'+ c +'_mean',  'hist_'+ c +'_max', 'hist_'+ c +'_std']\n    train = train.merge(hist_transt_amt_summ, how='left', on='card_id')\n    test = test.merge(hist_transt_amt_summ, how='left', on='card_id')\n    del(hist_transt_amt_summ)\n    \n    new_transt_amt_summ = new_trans.groupby(['card_id']).agg({c : ['count', 'sum', 'mean', 'max', 'std']}).reset_index()\n    new_transt_amt_summ.columns = ['card_id', 'new_'+ c +'_cnt', 'new_'+ c +'_sum', \n                                    'new_'+ c +'_mean',  'new_'+ c +'_max', 'new_'+ c +'_std']\n    train = train.merge(new_transt_amt_summ, how='left', on='card_id')\n    test = test.merge(new_transt_amt_summ, how='left', on='card_id')\n    del(new_transt_amt_summ)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f648647656be9542903c312b757e88ea66564d7a"},"cell_type":"code","source":"#train.sort_values(by='target', ascending=True)\n#C_ID_a4e600deef, C_ID_7e285a535a\ntrain[train.card_id.isin(['C_ID_a4e600deef', 'C_ID_7e285a535a'])].head().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ffcd05bd139927c0f4d44b945b0232e52501c8b"},"cell_type":"code","source":"ggplot(train, aes(x='hist_authorized_flag_mean', y='target')) +\\\ngeom_point(color='red', alpha=0.4, size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c28348625ca70de48bc5d7508fcde8567cc135e6"},"cell_type":"code","source":"ggplot(train[train.hist_purchase_amount_sum<2000000], aes(x='hist_purchase_amount_sum', y='target')) +\\\ngeom_point(color='red', alpha=0.4, size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3c0c45ff85ff82070b6b874a85786f9775bca7d"},"cell_type":"code","source":"train_corr = train.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c72cbfb3b4bb4954412cc3312cfb72583cd5c3b"},"cell_type":"code","source":"plt.figure(figsize=(8,13))\ntrain_corr['target'] = train_corr['target'].abs().sort_values(ascending=False)\ntrain_corr.drop('target', axis=0)['target'].plot(kind='barh')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6eefca8ff67b254a8445128dcf0a552b7503e8c"},"cell_type":"markdown","source":"Creating validation set for cross validation. \n* train set size : 80%\n* validation set size : 20%"},{"metadata":{"trusted":true,"_uuid":"5c14399b9e5d70327e2d7bfd45cd68260339374b"},"cell_type":"code","source":"train = train.fillna(0)\ntest = test.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a85129b08f5bb27adcd2a080930850245a5e8d3"},"cell_type":"code","source":"X_train = train.drop(['target', 'first_active_month', 'card_id'], axis=1, inplace=False)\ny_train = train.target\nX_test = test.drop([ 'first_active_month', 'card_id'], axis=1, inplace=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a48256e67f878c00ac69d973a364a9a112e78dbd"},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16a67532a254afc1679fe528365d192ec7d2123b"},"cell_type":"code","source":"print(X_train.shape, X_val.shape, X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c89acd4caea7cf32921c7975c0af02d9c650d79"},"cell_type":"markdown","source":"Let's run few algorithams with default/my favorite parameters\n\n**Linear Regression:**"},{"metadata":{"trusted":true,"_uuid":"46a3aae55c728a60aa8503882728bf943872652a"},"cell_type":"code","source":"lr = LinearRegression()\nlr.fit(X_train, y_train)\ny_pred = lr.predict(X_val)\nnp.sqrt(mean_squared_error(y_val, y_pred)) #3.8346","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ada8b6e7b6e6b4cdaf6ba542388aa1725582ed8"},"cell_type":"markdown","source":"Decision Tree Regression:\nTaking max depth value as 5 and minimum sample leaf as 10. I almost always choose this parameter for first traila with decision tree based on my past experience."},{"metadata":{"trusted":true,"_uuid":"3565975e86d44d4b23105960cb865ec3b97e38cd"},"cell_type":"code","source":"dtr = DecisionTreeRegressor(max_depth = 6, min_samples_leaf=10)\ndtr.fit(X_train, y_train)\ny_pred = dtr.predict(X_val)\nnp.sqrt(mean_squared_error(y_val, y_pred)) # 3.8272","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dcb5aadd1c4cb6a8d8222806117c63b57c12b33f"},"cell_type":"markdown","source":"**KNN Regression:**"},{"metadata":{"trusted":true,"_uuid":"3a78a86f24582486800bd525ecb40df969163053"},"cell_type":"code","source":"'''for i in range(1,30):\n    knn = KNeighborsRegressor(n_neighbors=i, n_jobs=12)\n    knn.fit(X_train, y_train)\n    y_pred = knn.predict(X_test)\n    print(i, np.sqrt(mean_squared_error(y_test, y_pred)) )\n'''    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"619f768e5005c87c179cd2d7960256ef404a7166"},"cell_type":"markdown","source":"**Random Forest Regression:**\n\nAgain with some of my favorite parameters for first submission"},{"metadata":{"trusted":true,"_uuid":"5dc8d7a2ca0e4e9347a4bcba68fa1e2e4b4c77f3"},"cell_type":"code","source":"'''rfr = RandomForestRegressor(max_depth=6, n_estimators=150, min_samples_leaf=5)\nrfr.fit(X_train, y_train)\nrf_pred = rfr.predict(X_val)\nrf_train_pred = rfr.predict(X_train)\nprint(np.sqrt(mean_squared_error(y_val, rf_pred))) # 3.8027 #3.854 on LB\nprint(np.sqrt(mean_squared_error(y_train, rf_train_pred))) # 3.7342\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ec184208cc1066d846a26fad5afbdf9cec0356d"},"cell_type":"code","source":"train_columns = X_train.columns\nlgb_train = lgb.Dataset(X_train, y_train, feature_name=list(train_columns))\nlgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train, feature_name=list(train_columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"654cf09b294b26ca8f76f0ba74ab9067f9cad8dd"},"cell_type":"code","source":"params = {\n    'objective': 'regression_l2',\n    'metric': { 'rmse'},\n    'num_leaves': 2000,\n    'learning_rate': 0.1,\n    'feature_fraction': 1,\n    'verbose': 0,\n    'max_depth' : 6,\n    'min_data_in_leaf' : 8\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"012f2be1451372645273aa4d5b477afc0540dfac"},"cell_type":"code","source":"gbm = lgb.train(params,\n                lgb_train,\n                num_boost_round=200000,\n                valid_sets=lgb_eval,\n                early_stopping_rounds=50) #3.76446","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ca322a7f38c16128e2b8cd31598d3d1b1521eaf"},"cell_type":"code","source":"test_lgbm_pred = gbm.predict(X_test, num_iteration = gbm.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96cb242673991d2cc393df4642831b7b1ccb9ac6"},"cell_type":"code","source":"submission = pd.DataFrame(test_lgbm_pred, index=X_test.index)\nsubmission.columns=['target']\nsubmission = pd.concat([test.card_id, submission], axis=1)\nsubmission.to_csv('submission_lgbm.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"196067ae82b8d212a77fb006e68c29a9d627240e"},"cell_type":"code","source":"lgb_fi = pd.DataFrame(gbm.feature_importance(), index=train_columns).reset_index()\nlgb_fi.columns = ['column','score']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"244a3bedd545cc78426b7851c810cf0f64a60c0e"},"cell_type":"code","source":"plotnine.options.figure_size = (14, 20)\nggplot(lgb_fi, aes(x='column', y='score')) +\\\ngeom_bar(stat='identity') +\\\ntheme(axis_text_x=element_text(rotation=90, hjust=1)) +\\\nplotnine.coord_flip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09cd443d3e607b3987002e1b821504d8e280cb5e"},"cell_type":"code","source":"lgb_fi.sort_values(by='score', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f65a2f30adb8e16c2c42ec586d71b0679353999"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}