{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd,numpy as np,matplotlib.pyplot as plt,seaborn as sns\nimport os\nimport warnings\nimport scipy\nwarnings.filterwarnings(\"ignore\")\npd.set_option(\"display.max_columns\",200)\nimport datetime\nimport gc\nimport xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Train data"},{"metadata":{"trusted":true,"_uuid":"8c0ed63fda516da3780a20a54c81e9601a152573"},"cell_type":"code","source":"# Reading training data\ntrain = pd.read_csv(\"/kaggle/input/train.csv\")\n# Making sure there is no duplicate card id\ntrain.shape[0] == train[\"card_id\"].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92d1703f9bf8bd4bfb1a76fed56959fc1bd1306e"},"cell_type":"code","source":"# Checking for Null values\ntrain.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38a3a1252083a022141766017563f7489755201b"},"cell_type":"code","source":"# break down first active month into year and month\ntrain[\"active_year\"],train[\"active_month\"] = list(zip(*train[\"first_active_month\"].apply(lambda x:x.split(\"-\"))))\n# unique values of features\ntrain.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59c78600bf3f0a95874fd237dd90ab24746e0f56"},"cell_type":"code","source":"# histogram of target\nplt.figure(figsize=(14,14))\nsns.distplot(train[\"target\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"127c75f35cbe0bcfe2198d352ca37c0fec34ae43"},"cell_type":"markdown","source":"**Few values of y are less than -30, apart from that distribution is normal and centered around 0**"},{"metadata":{"trusted":true,"_uuid":"d2bb843c1cd321ae126c1cbcb999396fd7e0ed0f"},"cell_type":"code","source":"# box plot of target based on features present\nfeature = [\"active_year\",\"active_month\",\"feature_1\",\"feature_2\",\"feature_3\"]\nf,ax=plt.subplots(1,5)\nf.set_figheight(8)\nf.set_figwidth(20)\nfor k,i in enumerate(feature):\n    sns.boxplot(x= i,y=\"target\",data = train,ax=ax[k])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1a930c7f1cb917a60993341bfe517cfd7337ad5"},"cell_type":"code","source":"# correlation heat map of features\ntrain[[\"active_year\",\"active_month\"]] = train[[\"active_year\",\"active_month\"]].astype(int)\ncorr = train.drop([\"card_id\",\"first_active_month\"],axis=1).corr()\nplt.figure(figsize =(10,6))\nsns.heatmap(corr,annot=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f30da3f637171e6e4ef64d7354f9575a50581c38"},"cell_type":"markdown","source":"** No significant correlation between target and predictors  **"},{"metadata":{"_uuid":"0575a8e1b28019b92cd0581e292a36635eab3cba"},"cell_type":"markdown","source":"# Test data"},{"metadata":{"trusted":true,"_uuid":"1ab8cf034763a44c3f497a74745069abcb9cd245"},"cell_type":"code","source":"# Reading test dataset\ntest = pd.read_csv(\"/kaggle/input/test.csv\")\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1d21f81be6c6896c42ca428c2fb4519af8e5ec2"},"cell_type":"code","source":"# checking for null values\ntest.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2305a607460bed8520e44152506655778f5b7da8"},"cell_type":"markdown","source":"** one value of first active month is missing. replace that with mode of training data first active month value **"},{"metadata":{"trusted":true,"_uuid":"8832f6c2a3711efe31dfd41ab31c09ec4757782b"},"cell_type":"code","source":"# replacing one null with mode of first active month of train set\nimpute = train[\"first_active_month\"].value_counts()[:1].index[0]\ntest = test.fillna(impute)\n\ntest[\"active_year\"],test[\"active_month\"] = list(zip(*test[\"first_active_month\"].apply(lambda x:x.split(\"-\"))))\ntest[[\"active_year\",\"active_month\"]] = test[[\"active_year\",\"active_month\"]].astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfddca8f708884d491f1feffceed99ef3d4b8e1a"},"cell_type":"code","source":"# created elapsed features using the following kernel\n#https://www.kaggle.com/tunguz/eloda-with-feature-engineering-and-stacking\ntrain[\"first_active_month\"] = pd.to_datetime(train[\"first_active_month\"])\ntrain['elapsed_time'] = (datetime.date(2018, 2, 1) - train['first_active_month'].dt.date).dt.days\ntest[\"first_active_month\"] = pd.to_datetime(test[\"first_active_month\"])\ntest['elapsed_time'] = (datetime.date(2018, 2, 1) - test['first_active_month'].dt.date).dt.days\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e66fe10463ba63f1bcd96b7d25dc312a64a1adff"},"cell_type":"markdown","source":"# Historical"},{"metadata":{"trusted":true,"_uuid":"aafc61895f6f5b78bb72062084f59414178ec12d"},"cell_type":"code","source":"# Reading historical transaction\nhistorical_trans =pd.read_csv(\"/kaggle/input/historical_transactions.csv\")\nhistorical_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"415c8e0090faecab80ab1e3ac4a89ad9bf734d61"},"cell_type":"code","source":"historical_trans.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"305b0fd04bfa91dc8489d464a6107050985c51b2"},"cell_type":"code","source":"print(historical_trans[\"installments\"].unique())\n# assuming -1 as missing data and 999 as wrong entry, replacing both with 0, i.e mode of remaining data\nhistorical_trans[\"installments\"] = historical_trans[\"installments\"].replace([-1,999],[0,0],inplace=False)\n\n# Created binary feature from installment \nhistorical_trans[\"has_installments\"] = historical_trans[\"installments\"].apply(lambda x: \"No\" if x==0 else \"Yes\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fdbe34fc2d85214838ced6bfb7c2e1502060f7dc"},"cell_type":"markdown","source":"# New Merchant data"},{"metadata":{"trusted":true,"_uuid":"f633a70e4e9a3ad00753dab67ec7e99905b9823d"},"cell_type":"code","source":"new_merchant_transactions = pd.read_csv(\"/kaggle/input/new_merchant_transactions.csv\")\nnew_merchant_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1e7d1facfc941a06354d11729a0291a48453357"},"cell_type":"code","source":"# assuming -1 as missing data and 999 as wrong entry, replacing both with 0, i.e mode of remaining data\nnew_merchant_transactions[\"installments\"] = new_merchant_transactions[\"installments\"].replace([-1,999],[0,0],inplace=False)\nnew_merchant_transactions[\"has_installments\"] = new_merchant_transactions[\"installments\"].apply(lambda x: \"No\" if x==0 else \"Yes\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75eee2dc091b8d5cb0ad6195b71544a22404800b"},"cell_type":"code","source":"# Imputing missing values \ndef impute(df):\n    df[\"merchant_id\"] = df[\"merchant_id\"].fillna(\"Missing_id\")\n    features = df.columns[df.isna().any()].tolist()\n    for i in features:\n        if df[i].dtype ==\"object\" or i == \"category_2\":\n            mode = df[i].value_counts()[:1].index[0]\n            df[i].fillna(mode,inplace=True)\n        else:\n            df[i].fillna(df[i].mean(),inplace=True)\n    return df\n\nnew_merchant_transactions = impute(new_merchant_transactions)\nhistorical_trans = impute(historical_trans)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8a4749e5f698ed3efdea9ef6a9e1a56af8d7918"},"cell_type":"markdown","source":"# Aggregation"},{"metadata":{"trusted":true,"_uuid":"57b10c568bda32cae09974336e15437297fd7ef8"},"cell_type":"code","source":"# mapping of binomial feature to (0,1)\nfor i in [\"authorized_flag\",\"category_1\"]:\n    new_merchant_transactions[i] = new_merchant_transactions[i].map({\"Y\":1,\"N\":0})\n    historical_trans[i] = historical_trans[i].map({\"Y\":1,\"N\":0})\n\nhistorical_trans.loc[:, 'purchase_date'] = pd.DatetimeIndex(historical_trans['purchase_date']).astype(np.int64) * 1e-9\nnew_merchant_transactions.loc[:, 'purchase_date'] = pd.DatetimeIndex(new_merchant_transactions['purchase_date']).astype(np.int64) * 1e-9\n\n# aggregation for numerical features\naggs = {'month_lag':[\"min\",\"max\",\"median\",'sum'],\n       'purchase_amount':['mean','var'],\n       'category_2':['min','max','median'],\n    'installments':['min','max','median','sum'],\n        \"authorized_flag\":['sum','mean'],\n        \"category_1\":['sum','mean'],\n        'merchant_id':[\"nunique\"],'merchant_category_id':[\"nunique\"]\n        }\n\nnew_columns_hist = [k + '_hist_' + agg for k in aggs.keys() for agg in aggs[k]]\nnew_columns_new = [k + '_new_' + agg for k in aggs.keys() for agg in aggs[k]]\n\n# new transaction data\naggs_merge2 = new_merchant_transactions.groupby(\"card_id\").aggregate(aggs)\naggs_merge2 = aggs_merge2.fillna(0) # card id with one entry will have nan variance.\naggs_merge2.columns = new_columns_new\naggs_merge2[\"new_purchase_date\"] = new_merchant_transactions.groupby(\"card_id\").aggregate({'purchase_date':[np.ptp]}).values\ndel new_merchant_transactions\ngc.collect()\n\n# past data\naggs_merge_1 = historical_trans.groupby(\"card_id\").aggregate(aggs)\naggs_merge_1 = aggs_merge_1.fillna(0) # card id with one entry will have nan variance.\naggs_merge_1.columns = new_columns_hist\naggs_merge_1[\"history_purchase_date\"] = historical_trans.groupby(\"card_id\").aggregate({'purchase_date':[np.ptp]}).values\ndel historical_trans\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af4ed48efe2f2978ce7facad09535eb10404144f"},"cell_type":"markdown","source":"# final merge"},{"metadata":{"trusted":true,"_uuid":"bfd86da4420affc4ea2799fe5bc87da4f3455898"},"cell_type":"code","source":"final_train = train.merge(aggs_merge_1,on=\"card_id\",how=\"left\")\nfinal_train = final_train.merge(aggs_merge2,on=\"card_id\",how=\"left\")\n\nfinal_test = test.merge(aggs_merge_1,on=\"card_id\",how=\"left\")\nfinal_test = final_test.merge(aggs_merge2,on=\"card_id\",how=\"left\")\ndel train,test\ngc.collect()\n# # imputation in merge dataset\nimputed_columns = final_train.columns[final_train.isna().any()].tolist()\nfor i in imputed_columns:\n    final_train[i].fillna(final_train[i].mean(),inplace=True)\n    final_test[i].fillna(final_train[i].mean(),inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"467a3dce63284d2e9ff0a64205d090d2482f84f0"},"cell_type":"markdown","source":"# Dropping features having only one unique value"},{"metadata":{"trusted":true,"_uuid":"18a362ef64de8f24b1b0199b983891ead837c4bf"},"cell_type":"code","source":"# finding features with only one unique value and dropping them\none_value_features =[]\nfor i in final_train.columns:\n    if final_train[i].nunique() ==1:\n        one_value_features.append(i)\n        \nfinal_train = final_train.drop(one_value_features,axis=1)\nfinal_test = final_test.drop(one_value_features,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87224644f40bdb3736783093ad141abf9f5e3402"},"cell_type":"code","source":"# Correlation Analysis of features\n# top 10 features correlated with target\ncorrelation = final_train.drop([\"first_active_month\",\"card_id\"],axis=1).corr()\ncorrelation[\"target\"].abs().sort_values(ascending=False)[:10]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d64e53231f8bdaecc5c9bfc7b1c1a0f16ed2145f"},"cell_type":"code","source":"# calculating correlation of all features and defining them \"important\" based on their P-value\npairwise_corr=[]\nvalue=[]\ncontinuous_columns=[]\nfor i in final_train.drop([\"target\",\"card_id\",\"first_active_month\"],axis=1).columns:\n    c, p = scipy.stats.pearsonr(final_train[i],final_train[\"target\"])\n    continuous_columns.append(i)\n    value.append(p)\n    pairwise_corr.append(c)\ndf=pd.DataFrame({\"column\":continuous_columns,\"corr_value\":pairwise_corr,\"p-value\":value})\ndf[\"importance\"]=df[\"p-value\"].apply(lambda x:\"important\" if x<0.05 else \"not important\")\nplt.figure(figsize=(20,10))\nplt.title(\"p value of predictor with target variable\")\nplt.xticks(rotation=\"vertical\")\nsns.stripplot(x=\"column\",y=\"p-value\",hue=\"importance\",data=df,size=10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"175a7033df3b2f29969b7964b220e33d2739fb7b"},"cell_type":"markdown","source":"# Splitting Data into train/validation/test set"},{"metadata":{"trusted":true,"_uuid":"d5318f5bbeb9ca8da137b92690269d8f9c60f565"},"cell_type":"code","source":"y_train = final_train[\"target\"]\nfinal_train = final_train.sample(frac=1, random_state = 7)\nx_train = final_train.drop([\"target\",\"card_id\",\"first_active_month\"],axis=1)\nx_test = final_test.drop([\"card_id\",\"first_active_month\"],axis=1)\nfrom sklearn.model_selection import train_test_split as tts\nTrn_x,val_x,Trn_y,val_y = tts(x_train,y_train,test_size =0.1,random_state = 7)\ntrn_x , test_x, trn_y, test_y = tts(Trn_x , Trn_y, test_size =0.1, random_state = 7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91eeab619bdd1dcdf84f9041c5d5e4fcfc7597ab"},"cell_type":"code","source":"# converting into xgb DMatrix\nTrain = xgb.DMatrix(trn_x,label = trn_y)\nValidation = xgb.DMatrix(val_x, label = val_y)\nTest = xgb.DMatrix(test_x)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"127411cfd0279a2c32c9a7b9b3c1e0233f350093"},"cell_type":"code","source":"params = {\"booster\":\"gbtree\",\"eta\":0.1,'min_split_loss':0,'max_depth':6,\n         'min_child_weight':1, 'max_delta_step':0,'subsample':1,'colsample_bytree':1,\n         'colsample_bylevel':1,'reg_lambda':1,'reg_alpha':0,\n         'grow_policy':'depthwise','max_leaves':0,'objective':'reg:linear','eval_metric':'rmse',\n         'seed':7}\nhistory ={}  # This will record rmse score of training and test set\neval_list =[(Train,\"Training\"),(Validation,\"Validation\")]\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb0927bf86a232936f4d3bc7fd08c1f42f5c105d"},"cell_type":"markdown","source":"# Training the model"},{"metadata":{"trusted":true,"_uuid":"bf94e1a4b4cac2359acd3500ef4d68438fedc324"},"cell_type":"code","source":"clf = xgb.train(params, Train, num_boost_round=119, evals=eval_list, obj=None, feval=None, maximize=False, \n          early_stopping_rounds=40, evals_result=history)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0fdb5a83b6292c238998e02876e5072183e29f2e"},"cell_type":"markdown","source":"# Evaluation of History of model"},{"metadata":{"trusted":true,"_uuid":"b9121aea07b4b0868b6282acb08865d952989987"},"cell_type":"code","source":"# dataframe of progress\nf,ax=plt.subplots(1,1)\nf.set_figheight(10)\nf.set_figwidth(20)\ndf_performance=pd.DataFrame({\"train\":history[\"Training\"][\"rmse\"],\"test\":history[\"Validation\"][\"rmse\"]}).reset_index(drop=False)\nsns.pointplot(ax=ax,y=\"train\",x=\"index\",data=df_performance,color=\"r\")\nsns.pointplot(ax=ax,y=\"test\",x=\"index\",data=df_performance,color=\"g\")\nax.legend(handles=ax.lines[::len(df_performance)+1], labels=[\"Train\",\"Test\"])\nplt.xlabel('iterations'); plt.ylabel('logloss value'); plt.title('learning curve')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efa85d45716f33fc63614684811b549b1a08f829"},"cell_type":"markdown","source":"# Plotting importance score of features from xgboost model (ordered by their significance)"},{"metadata":{"trusted":true,"_uuid":"bce45bbda4e1a60ce367cff11edc268530c09312"},"cell_type":"code","source":"score=clf.get_score(importance_type=\"gain\")\ndf=pd.DataFrame({\"feature\":list(score.keys()),\"score\":list(score.values())})\ndf=df.sort_values(by=\"score\",ascending=False)\nplt.figure(figsize=(20,20))\nplt.xticks(rotation=\"vertical\")\nsns.barplot(x=\"feature\",y=\"score\",data=df,orient=\"v\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19bc5b10c3b5e54edc20fc1a72679decaae52368"},"cell_type":"code","source":"# Checking rmse on test set (kept during data splitting)\nfrom sklearn.metrics import mean_squared_error as mse\npred_test = clf.predict(Test)\nscore = mse(test_y , pred_test)\nprint(np.sqrt(score))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ee7944806505a2681c8a37bfaa7a4ef7e1a26d1e"},"cell_type":"markdown","source":"# Making prediction on actual test data"},{"metadata":{"trusted":true,"_uuid":"5829c6500fa3a651fd5c6e8e9ec9423b6cb0aceb"},"cell_type":"code","source":"prediction = clf.predict(xgb.DMatrix(x_test))\ndf_sub=pd.DataFrame()\ndf_sub[\"card_id\"] = final_test[\"card_id\"].values\ndf_sub[\"target\"] = np.ravel(prediction)\ndf_sub[[\"card_id\",\"target\"]].to_csv(\"new_submission.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}