{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d81224653e7685b55dd063e9de6ce0be9e7fde9"},"cell_type":"markdown","source":"### Goal of this notebook is to do some basic EDA of Elo-Recommendations Data and start developing basic models, step by step.  \n\n**This notebook will be in progress at least till 12/15/2018**. So please excuse the typos and other rough edges. "},{"metadata":{"_uuid":"698807dd5ffe090fcac6ceb9de46e47b294fd40d"},"cell_type":"markdown","source":"### Let us first read the input data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')\ndf_test = pd.read_csv('../input/test.csv')\ndf_merchants = pd.read_csv('../input/merchants.csv')\ndf_hist_trans = pd.read_csv('../input/historical_transactions.csv')\ndf_new_merchant_trans = pd.read_csv('../input/new_merchant_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"38f757a08f1014e8e1dd76a7d446df7c166d98da"},"cell_type":"markdown","source":"### Display the data frames with titles. "},{"metadata":{"trusted":true,"_uuid":"6c3c47347efc7855f1313b8726b66fcdf40c8d5f"},"cell_type":"code","source":"print(\"Training Data Sample\");display(df_train.head())\nprint(\"Test Data Sample\");display(df_test.head())\nprint(\"Merchant Data Sample\");display(df_merchants.head())\nprint(\"Historical Transactions Sample\");display(df_hist_trans.head())\nprint(\"New Merchant Transactions Sample\");display(df_new_merchant_trans.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59f08d2f771089785e69a8a35742ddc6218a5e7f"},"cell_type":"markdown","source":"### Let us first understand the training data & test data"},{"metadata":{"trusted":true,"_uuid":"90d4d52b228d0e5bcbf57ba718b8f40759bd8bfb"},"cell_type":"code","source":"df_test.dtypes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d156878706f9da58226fbb027715e0ec9195739a"},"cell_type":"markdown","source":"It looks like the first_active_month is stored as a string type. Let us change it to datetime format"},{"metadata":{"trusted":true,"_uuid":"30672825eeef585cedef5dd4c91a2511d7ce2881"},"cell_type":"code","source":"df_train['first_active_month'] = pd.to_datetime(df_train['first_active_month'])\ndf_test['first_active_month'] = pd.to_datetime(df_test['first_active_month'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad2e61be11032b4c444663df16b4231dae097e4f"},"cell_type":"markdown","source":"First, let us look at the training and test data distributions to check if they are similar. "},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"55fe75dd61119133d89ef10dfd67783384bc4196"},"cell_type":"code","source":"f, axes = plt.subplots(1, 2, figsize=(12,6))\nsns.distplot( df_train.feature_1,ax=axes[0], kde = False, color = 'green', bins=10).set_title(\"Train Data\")\n\nsns.distplot( df_test.feature_1,ax=axes[1], kde = False, color = 'red', bins=10).set_title(\"Test Data\") \naxes[0].set(ylabel='Card Counts')\nf.suptitle('feture_1 Distributions: Training Data and Test Data')\naxes[0].set_xticks(np.arange(1,6,1))\naxes[1].set_xticks(np.arange(1,6,1))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"ce8af84b99225da0bafc728a4361199e364de2c9"},"cell_type":"code","source":"f, axes = plt.subplots(1, 2, figsize=(12,6))\nsns.distplot( df_train.feature_2,ax=axes[0], kde = False, color = 'green', bins=10).set_title(\"Train Data\")\n\nsns.distplot( df_test.feature_2,ax=axes[1], kde = False, color = 'red', bins=10).set_title(\"Test Data\") \naxes[0].set(ylabel='Card Counts')\nf.suptitle('feture_2 Distributions: Training Data and Test Data')\naxes[0].set_xticks(np.arange(1,4,1))\naxes[1].set_xticks(np.arange(1,4,1))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"fc21bab773f23e7dca37a61578efeccccf25492b"},"cell_type":"code","source":"f, axes = plt.subplots(1, 2, figsize=(12,6))\nsns.distplot( df_train.feature_3,ax=axes[0], kde = False, color = 'green', bins=10).set_title(\"Train Data\")\n\nsns.distplot( df_test.feature_3,ax=axes[1], kde = False, color = 'red', bins=10).set_title(\"Test Data\") \naxes[0].set(ylabel='Card Counts')\nf.suptitle('feture_3 Distributions: Training Data and Test Data')\naxes[0].set_xticks(np.arange(1,2,1))\naxes[1].set_xticks(np.arange(1,2,1))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"d8d99c799ecbb9361cb008d790cb906012ccb704"},"cell_type":"code","source":"f, axes = plt.subplots(1, 2, figsize=(12,6))\ndf_test['first_active_month'] = df_test['first_active_month'] .fillna(axis=0, method='ffill')\nsns.distplot( df_train.first_active_month,ax=axes[0], kde = False, color = 'green', bins=20).set_title(\"Train Data\")\nsns.distplot( df_test.first_active_month,ax=axes[1], kde = False, color = 'red', bins=20).set_title(\"Test Data\") \naxes[0].set(ylabel='Card Counts')\nf.suptitle('first_active_month Distributions: Training Data and Test Data')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e7971de71fe7ca1bd8decaa0851d3631b43f5eb"},"cell_type":"markdown","source":"### It can be seen that feature_1, feature_2, feature_3 and first_active month have similar distributions in the train and test data.\n\n### Now, let us use a the seaborn pairplot to quickly check if any of the input variables in the training data are clearly correlated with the target variable. "},{"metadata":{"trusted":true,"_uuid":"68a3ca3329f57c8fa38e7fd43c661d2ee5afa144"},"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.pairplot(df_train.loc[:,df_train.columns != 'card_id'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db5cb78c7465c03d2fa56628d23de5b2e5ac42cd"},"cell_type":"markdown","source":"From the above figure, it is not immediately clear if any of the feature_1, feature_2, feature_3 are correlated with the target variable. \n\n### Let us start with a linear model\n\nSince time stamps are more dificult to work with for linear models, let us start convert the first_active_moth feature, into a customer age feaure, which just returns the number of days since the customer first registered. "},{"metadata":{"trusted":true,"_uuid":"fdb1b3e39f817d49a8f5f15bb96708f8a1147de8"},"cell_type":"code","source":"latest_date = max(df_train['first_active_month'])\ndf_train['customer_age'] = (latest_date - df_train['first_active_month']).astype('timedelta64[D]')\ndf_test['customer_age'] = (latest_date - df_test['first_active_month']).astype('timedelta64[D]')\nsns.pairplot(df_train.loc[:,df_train.columns != 'card_id'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8de8eff4f7992039622f6d5bd2f81bc7ebba0cda"},"cell_type":"markdown","source":"**From the above, it looks like the customer age feature ma have some correlation with the target variable after all. **\n\n**Let us generate a linear model and submit it, as a first step. **"},{"metadata":{"trusted":true,"_uuid":"eb52750f47f07e9dd9b71dbe8024bba57c9e9353"},"cell_type":"code","source":"# split the training set into training set and validation set \n# We are not going to do K-fold cross validation for now. We will do that in a later model\n\nX_train, X_val, y_train, y_val = train_test_split(df_train[['feature_1','feature_2','feature_3','customer_age']],df_train['target'], test_size=0.1, random_state=42)\n#X_train.shape, X_val.shape, y_train.shape, y_val.shape\nlinear_model = LinearRegression().fit(X_train,y_train)\ny_pred = linear_model.predict(X_val)\nprint(\"Root Mean squared error: %.2f\" % np.sqrt(mean_squared_error(y_val, y_pred)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b70b35dab4fc8854becb18b5040a4d8f9469e136"},"cell_type":"markdown","source":"## Submit the best model so far"},{"metadata":{"trusted":true,"_uuid":"4aeec9685089408e7585d474d8267ae0a0996012"},"cell_type":"code","source":"y_test = linear_model.predict(df_test[['feature_1', 'feature_2', 'feature_3','customer_age']])\nlinear_submission = pd.DataFrame({\"card_id\": df_test[\"card_id\"].values})\nlinear_submission['target'] = y_test\nlinear_submission.to_csv('linear_submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e87431093b49af6919db7ae631ba50f79c74bf64"},"cell_type":"markdown","source":"<font color='red'>In progress. More to come.. </font>"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}