{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_data=pd.read_csv(r'../input/train.csv')\ntest_data=pd.read_csv(r'../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9ceef1189993fb5b90e805ffaa04db91d104bc6"},"cell_type":"code","source":"!../input/Data_Dictionary.xlsx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a4b6d8ab77ccd3abac67c1283302dce023f53e1"},"cell_type":"code","source":"dict1=pd.read_excel(r'../input/Data_Dictionary.xlsx')\ndict1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5337d227a301ab2c1703df2acd8cb13805d2d7a9"},"cell_type":"code","source":"train_data.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d23e4b9c4f13a232ee5e2a3b378b5528d5275945"},"cell_type":"code","source":"train_data.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a67a69d80c7ac022016a2a57513cfe8503fc7e7"},"cell_type":"code","source":"train_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33e5bf735ee20ff0066335222803d17a447c029f"},"cell_type":"code","source":"train_data.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf8f4e3c1ff2c911d9d345fb81964ccf442bf129"},"cell_type":"code","source":"train_data.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f78eb1b03751f47d471e2a66971c1073ce9ddab"},"cell_type":"code","source":"target_col=train_data['target']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2a289d5937e6a0ee5de6b09df01007fc84d32da"},"cell_type":"code","source":"plt.figure(figsize=(8,6),dpi=80)\nsns.violinplot(x=train_data['feature_1'],y=train_data['target'],data=train_data,)\nplt.xlabel('Target',size=18,color='r')\nplt.ylabel('Feature_1',size=18,color='r')\nplt.title('Feature_1 Vs Target',size=20,color='blue')\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01d207e61f165f91fbb8b002013609e94f1e242b"},"cell_type":"code","source":"plt.figure(figsize=(8,6),dpi=80)\nsns.violinplot(x=train_data['feature_2'],y=train_data['target'],data=train_data,)\nplt.xlabel('Target',size=18,color='r')\nplt.ylabel('Feature_2',size=18,color='r')\nplt.title('Feature_2 Vs Target',size=20,color='blue')\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40a90385c3712d9cd143fdb98aa9946d180ffd3f"},"cell_type":"code","source":"plt.figure(figsize=(8,6),dpi=80)\nsns.violinplot(x=train_data['feature_3'],y=train_data['target'],data=train_data)\nplt.xlabel('Target',size=18,color='r')\nplt.ylabel('Feature_3',size=18,color='r')\nplt.title('Feature_3 Vs Target',size=20,color='blue')\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef10518b094e42754e8dca38bccfddc0223a72c3"},"cell_type":"code","source":"plt.figure(figsize=(8,6))\nsns.distplot(train_data['target'],bins=50,hist=True,color='#F71212',kde=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e5b0c73940be742b1dc01d63e5a98422d5fbc89"},"cell_type":"markdown","source":"As we can see that some targets are below** (-)30** lets check how many are there "},{"metadata":{"trusted":true,"_uuid":"b08fee0cb1487cc302b7d25f333c588055f3ce6e"},"cell_type":"code","source":"print('there are {0} sample in target below -30'.format(train_data.loc[train_data.target<-30].shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bada384d227fb83362aa153cd8a45dbe6c2ee039"},"cell_type":"code","source":"feature1=train_data['feature_1'].value_counts().sort_index(ascending=False)\nsns.barplot(x=feature1.index,y=feature1.values,ci=100,color='#FFFF00')\nplt.title('Feature_1',color='red',size=18);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6eb4b887cced29ce96790607597adb63e9e9d9c"},"cell_type":"code","source":"feature2=train_data['feature_2'].value_counts().sort_index(ascending=False)\nsns.barplot(x=feature2.index,y=feature2.values,ci=100,color='#00FF7F')\nplt.title('Feature_2',color='red',size=18);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72f20b5e1094e4203bab88b184ea3ec9c5632b37"},"cell_type":"code","source":"feature3=train_data['feature_3'].value_counts().sort_index(ascending=False)\nsns.barplot(x=feature3.index,y=feature3.values,ci=100,color='#556B2F')\nplt.title('Feature_3',color='red',size=18);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f117f38ccd63618503d99abafc94a1ec43a35db"},"cell_type":"markdown","source":"As we can see that different feature have different distribution of data according to target \n1. feature_1 value varied from 1 to 5 in target \n2. feature_1 value varied from 1 to 3 in target \n3. feature_1 value varied from 0 to 1 in target "},{"metadata":{"trusted":true,"_uuid":"cd01220fa9add2f233487638f829e868a773acf0"},"cell_type":"code","source":"train_active_months=train_data['first_active_month'].value_counts().sort_index(ascending=False)\ntest_active_months=test_data['first_active_month'].value_counts().sort_index(ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd7249ce0bf4815a146c53f9a0edc1ce912bbb0d"},"cell_type":"code","source":"data = [go.Scatter(x=train_active_months.index, y=train_active_months.values, name='train',opacity=1), \n        go.Scatter(x=test_active_months.index, y=test_active_months.values, name='test',opacity=1)]\nlayout = go.Layout(dict(title = \"Counts of first active month\",\n                  xaxis = dict(title = 'Month'),\n                  yaxis = dict(title = 'Count'),\n                  ),legend=dict(\n                orientation=\"v\"))\npy.iplot(dict(data=data, layout=layout))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05af8e8ae8fc2948d0fc6338631e3210bf04f093"},"cell_type":"markdown","source":"As we can see that both train and test active_month data follow the same trends and thats great !!"},{"metadata":{"trusted":true,"_uuid":"8c45781ca41f055db3044963ede75ff497ee2896"},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"441e8ca637bb0a9776cd9fd75718cd5c4ad84f58"},"cell_type":"code","source":"test_data.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abcbc3367705073b873c3aee83dc48304abdb918"},"cell_type":"code","source":"test_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8133fdcb9104249ac30b60651bf5ab279748ae32"},"cell_type":"code","source":"test_data.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2f27767b2dff8f9adc1b75e7267f401bf5ae4c37"},"cell_type":"markdown","source":"Ahh!!! one missing value in first_active_month in test data "},{"metadata":{"trusted":true,"_uuid":"82cca05bc606eb6ec2f609066e62201dbe4ddbde"},"cell_type":"code","source":"historical_data=pd.read_csv(r'../input/historical_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d019fe1d77b503266f28c0fd0ffc1b475e42a41"},"cell_type":"code","source":"historical_data.head(20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6f56fcded63cdcac6f0db25088261b9b44a50d2"},"cell_type":"markdown","source":" *** The field descriptions are as follow:**\n1. card_id - Card identifier\n2. month_lag - month lag to reference date\n1. category_1 - anonymized category\n1. category_2 - anonymized category\n1. category_3 - anonymized category\n3. purchase_date - Purchase date\n4. authorized_flag - 'Y' if approved, 'N' if denied\n1. installments - number of installments of purchase\n1. merchant_category_id - Merchant category identifier (anonymized )\n1. merchant_id - Merchant identifier (anonymized)\n1. purchase_amount - Normalized purchase amount\n1. city_id - City identifier (anonymized )\n1. state_id - State identifier (anonymized )\n1.  subsector_id - Merchant category group identifier (anonymized )\n"},{"metadata":{"trusted":true,"_uuid":"a155549da1128410294e92e0a66d5a49e71dba8b"},"cell_type":"code","source":"historical_data.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"539246fd189bdb0468d28f318be6e1044d658eeb"},"cell_type":"code","source":"historical_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"690a88b1569327f34ff88562ac8bf70902ce4dcc"},"cell_type":"code","source":"historical_data.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76a36facdf6ba14db56300a71340ca91124f476c"},"cell_type":"code","source":"historical_data['installments'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74a09169025f325549a3bbc40b057bda151ef34e"},"cell_type":"code","source":"plt.figure(figsize=(8,6),dpi=100)\ninstall=historical_data['installments'].value_counts().sort_index(ascending=False)\nsns.barplot(x=install.index,y=install.values,ci=100,color='#900C3F')\nplt.title('installment detail',color='red',size=18);\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7e982f71d1840fceaf86f276dfd5e879d4f5604"},"cell_type":"code","source":"trai=historical_data.groupby(['installments'])['authorized_flag'].value_counts()\ntrai.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4609906196174d3b8baa1e5f5bf94cb8b4e08415"},"cell_type":"code","source":"new_merchant_data=pd.read_csv(r'../input/new_merchant_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18033fc20e41b2141a0a0e004b457c083eb9b314"},"cell_type":"code","source":"new_merchant_data.head(120)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8436c12461531c3195f30ac0d8354d2c1d54fce9","scrolled":true},"cell_type":"code","source":"new_merchant_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44fef1ff620f52b6a9c3ed4b0e8e2a0be6e93399"},"cell_type":"code","source":"new_merchant_data.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4d37278c8be75da6834357d8ca14e013fc01c44"},"cell_type":"markdown","source":"category_2 follow trends of **->** **1.0 **\ncategory_3 follow ->A,B or C"},{"metadata":{"trusted":true,"_uuid":"68acd817d0bb55908a80ead85ea42ed8986985d2"},"cell_type":"code","source":"new_merchant_data['category_2'].fillna(value=0.1,inplace=True)\nnew_merchant_data['category_3'].fillna('B',inplace=True)\nnew_merchant_data['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abaafeb714ad87273749d8be6d7f60c5de6bc199"},"cell_type":"code","source":"new_merchant_data['authorized_flag']=new_merchant_data['authorized_flag'].apply(lambda x:1 if x=='Y'else 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45af8c319355eb3bec0f345684fcf6bf05449bf9"},"cell_type":"code","source":"new_merchant_data['authorized_flag'].value_counts().plot(kind='bar',title='authorize_flag value',color='red')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3d8f46a24cba2f5a3bb3be12a6c0c7982180681"},"cell_type":"markdown","source":"All Transcation looks authorized "},{"metadata":{"trusted":true,"_uuid":"8a558f1f6b20797702fb032b1c3b421319fd342e"},"cell_type":"code","source":"card_total=new_merchant_data.groupby(['card_id'])['purchase_amount'].sum().sort_values()\ncard_total.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9da9e011618186c6c148737db6f0f7974d242a82"},"cell_type":"code","source":"card_total.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5099a4448d2ac9263937701c5f4b9f78b30f3d0d"},"cell_type":"code","source":"hist = historical_data.groupby([\"card_id\"])\nhist= hist[\"purchase_amount\"].size().reset_index()\nhist.columns = [\"card_id\", \"hist_transactions\"]\ntrain_data = pd.merge(train_data,hist, on=\"card_id\", how=\"left\")\ntest_data = pd.merge(test_data,hist, on=\"card_id\", how=\"left\")\nhist.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"877f3ff239a2969b25e392927a9089b722c56155"},"cell_type":"code","source":"hist = historical_data.groupby([\"card_id\"])\nhist= hist[\"purchase_amount\"].agg(['sum','mean','max','min','std']).reset_index()\nhist.columns=['card_id','sum_hist_tran','mean_hist_tran','max_hist_tran','min_hist_tran','std_hist_tran']\ntrain_data=pd.merge(train_data,hist,on='card_id',how='left')\ntest_data=pd.merge(test_data,hist,on='card_id',how='left')\nhist.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6348ff0fddbb52e66076be1ad9a6560ca8044437"},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b232cc6fadde334eea4ff0c0236b29ed63a203e9"},"cell_type":"code","source":"merchant = new_merchant_data.groupby([\"card_id\"])\nmerchant= merchant[\"purchase_amount\"].size().reset_index()\nmerchant.columns = [\"card_id\", \"merchant_transactions\"]\ntrain_data = pd.merge(train_data,merchant, on=\"card_id\", how=\"left\")\ntest_data = pd.merge(test_data,merchant, on=\"card_id\", how=\"left\")\nmerchant.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cb243b11a0051adb0346329915b19d3fc40bc4a"},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b778fad24cf434d59d50dcd6784ff3cee30f78b5"},"cell_type":"code","source":"merchant= new_merchant_data.groupby([\"card_id\"])\nmerchant= merchant[\"purchase_amount\"].agg(['sum','mean','max','min','std']).reset_index()\nmerchant.columns=['card_id','sum_merchant_tran','mean_merchant_tran','max_merchant_tran','min_merchant_tran','std_merchant_tran']\ntrain_data=pd.merge(train_data,merchant,on='card_id',how='left')\ntest_data=pd.merge(test_data,merchant,on='card_id',how='left')\nmerchant.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ef8d5e1dcd4c069cd10c409a8f02b2a53311398"},"cell_type":"code","source":"train_data['first_active_month']=pd.to_datetime(train_data['first_active_month'])\ntest_data['first_active_month']=pd.to_datetime(train_data['first_active_month'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59a90c4ecee7f587379eca1f0f8471b4d3794f56"},"cell_type":"code","source":"train_data.drop('target',axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ac83b3ccb44c591ad2e1fba75a9b6c7d1b6598b"},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d08ba6e2092113273b8e7ebf93f41ea946ad7aa"},"cell_type":"code","source":"from sklearn import model_selection\nfrom sklearn import preprocessing\nfrom sklearn import metrics\nimport lightgbm as lgb\ntrain_data[\"year\"] = train_data[\"first_active_month\"].dt.year\ntest_data[\"year\"] = test_data[\"first_active_month\"].dt.year\ntrain_data[\"month\"] = train_data[\"first_active_month\"].dt.month\ntest_data[\"month\"] = test_data[\"first_active_month\"].dt.month\n\ncols_to_use = [\"feature_1\", \"feature_2\", \"feature_3\", \"year\", \"month\", \n               \"hist_transactions\", \"sum_hist_tran\", \"mean_hist_tran\", \"std_hist_tran\", \n               \"min_hist_tran\", \"max_hist_tran\",\n               \"merchant_transactions\", 'sum_merchant_tran','mean_merchant_tran',\n               'max_merchant_tran','min_merchant_tran','std_merchant_tran',\n              ]\n\ndef run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\",\n        \"num_leaves\" : 30,\n        \"min_child_weight\" : 50,\n        \"learning_rate\" : 0.05,\n        \"bagging_fraction\" : 0.7,\n        \"feature_fraction\" : 0.7,\n        \"bagging_freq\" : 5,\n        \"bagging_seed\" : 2019,\n        \"verbosity\" : -1\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    evals_result = {}\n    model = lgb.train(params, lgtrain, 10000, valid_sets=[lgval], \n        early_stopping_rounds=100, verbose_eval=100, evals_result=evals_result)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, evals_result\n\ntrain_X = train_data[cols_to_use]\ntest_X = test_data[cols_to_use]\ntrain_y = target_col.values\n\npred_test = 0\nkf = model_selection.KFold(n_splits=5, random_state=2019, shuffle=True)\nfor dev_index, val_index in kf.split(train_data):\n    dev_X, val_X = train_X.loc[dev_index,:], train_X.loc[val_index,:]\n    dev_y, val_y = train_y[dev_index], train_y[val_index]\n    \n    pred_test_tmp, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X)\n    pred_test += pred_test_tmp\npred_test /= 5.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"32cd18206d2b1c020cecdb412ad8e9431a8d7c92"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,10))\nlgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax,color='red')\nax.grid(False)\nplt.title(\"LightGBM - Feature Importance\", fontsize=15,color='r')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddc5caecb6074d9dac99907b4aa43c58e89d4b8a"},"cell_type":"code","source":"submission_data = pd.DataFrame({\"card_id\":test_data[\"card_id\"].values})\nsubmission_data[\"target\"] = pred_test\nsubmission_data.to_csv(\"baseline_lgb1.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e31c616c68efae3b231727b217f9a607d5e3db7a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"822d3381e251c4a6401b4e7a650c5f10f0134d7f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"108416c250d461911d2e617d2f6c6fb31d403768"},"cell_type":"markdown","source":"****Many more to come stay tuned!!!**\n\n**DON'T FORGET TO UPVOTE :)****"},{"metadata":{"trusted":true,"_uuid":"38869674e9d14f5ae83dcdb7d67d5471b39f7dd0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b499a480a3862698da926540a0fd8307c8afda2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75bb3996e87d2f7e9d4a0590f169276b78abc2cd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ed109cf5ecd05cccdd73502385143d046a67084"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}