{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nsns.set(rc={'figure.figsize':(7 ,5)})","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\nmerchants_df = pd.read_csv(\"../input/merchants.csv\")\nhistorical_transactions_df = pd.read_csv(\"../input/historical_transactions.csv\")\nnew_merchant_transactions_df = pd.read_csv(\"../input/new_merchant_transactions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f77528e7992af87ddb2699212bbff7c62cbaf0fc"},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true,"_uuid":"e39c8728d70f38e466ded2cf8e3557296af2c146"},"cell_type":"code","source":"print(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3dd8d696c4c20ae5aae8702f21dcff60b2e1d07"},"cell_type":"code","source":"train_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63c93d96453387c7d2e355afc60caf42cea177e0"},"cell_type":"code","source":"#There are some obvious outliers in 'target'\nsns.distplot(train_df.target.values, kde=False)\nprint (f'{train_df[train_df.target < -10].target.shape[0]} outliers')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efd3d1fc85e73bc8dc3d8f3a8860062afa898212"},"cell_type":"code","source":"train_df = train_df[train_df.target > -10]\nsns.pairplot(train_df[['feature_1', 'feature_2', 'feature_3', 'target']])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8926686439b50fd26d1a42c3d9b58c63def0cbf6"},"cell_type":"markdown","source":"# Merchants\n\nmerchants can be joined with the transaction sets to provide additional merchant-level information."},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"8af866d05a306911c9d6a543cb9579df9c81b2c5"},"cell_type":"code","source":"print(merchants_df.shape)\nmerchants_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c57df3ff5994db45b7f8d42cee8531303dd825e"},"cell_type":"code","source":"#There are some NaN values\nmerchants_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"c7fe8296e466c9bc9966ba578ce6caa1b727dd10"},"cell_type":"code","source":"merchants_df.dropna(inplace=True)\nnumerical_cols = ['numerical_1', 'numerical_2', 'avg_sales_lag3',\n                  'avg_purchases_lag3','active_months_lag3',\n                  'avg_sales_lag6', 'avg_purchases_lag6',\n                  'active_months_lag6', 'avg_sales_lag12', \n                  'avg_purchases_lag12', 'active_months_lag12']\n\nmerchants_df[numerical_cols].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"6ed8030ef6ad259b31b55d93d26e6b4e0a3b99d7"},"cell_type":"code","source":"cat_cols = [ 'category_1', 'category_2',  'category_4', \n          'most_recent_purchases_range', 'most_recent_sales_range',\n           'merchant_group_id', 'merchant_category_id', 'subsector_id']\n\nfor col in cat_cols:\n    cat_num = merchants_df[col].value_counts().index.shape[0]\n    print(f'{cat_num} unique values in {col}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06dac1e38249d7841aade03b464f81d1723a0819"},"cell_type":"code","source":"print('Almost 14% of values  belong to merchant group #35')\nrows = merchants_df.shape[0]\nmerchants_df.merchant_group_id.value_counts()[:5]/rows*100","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"137cee65194766d3bd9306f16b816113af11100f"},"cell_type":"markdown","source":"# Transactions\n\n**historical_transactions.csv**  - up to 3 months' worth of historical transactions for each card_id"},{"metadata":{"trusted":true,"_uuid":"1d584607a208c5676e149387149d2bbef1614640"},"cell_type":"code","source":"#Missing values\nhistorical_transactions_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3a7ac153fbcbb821c009e7805bbb737000d829a"},"cell_type":"code","source":"print(historical_transactions_df.shape)\nhistorical_transactions_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b6477193d97ec488bf63e397a579db3924971a4"},"cell_type":"code","source":"cat_cols = [ 'authorized_flag', 'card_id', 'city_id', 'category_1', \n             'category_3', 'merchant_category_id', 'merchant_id', \n             'category_2', 'state_id', 'subsector_id']\n\nfor col in cat_cols:\n    cat_num = historical_transactions_df[col].value_counts().index.shape[0]\n    print(f'{cat_num} unique values in {col}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"9b7880552425a029204b49e699b738a332ad4d97"},"cell_type":"code","source":"#example of continuous data on single card_id\nidx = np.random.choice(historical_transactions_df.card_id)\ndf = historical_transactions_df[historical_transactions_df.card_id == idx]\nsns.pairplot(df[['month_lag', 'purchase_amount']])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1757e262760b931fa7c1f37fadd2e8a45fe968f"},"cell_type":"markdown","source":"# New Merchant Transactions\n\n**historical_transactions.csv** and **new_merchant_transactions.csv** are designed to be joined with **train.csv**, ** test.csv**, and **merchants.csv**. They contain information about transactions for each card."},{"metadata":{"trusted":true,"_uuid":"8c9af8cb52f114943aa9a3516457d658acbb249e"},"cell_type":"code","source":"print(new_merchant_transactions_df.shape)\nnew_merchant_transactions_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06dcae257839db6dc80c338644812af7fc29f89f"},"cell_type":"code","source":"new_merchant_transactions_df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97808833df7f3bd14c03291a32c95545d7d52058"},"cell_type":"code","source":"cat_cols = [ 'authorized_flag', 'card_id', 'city_id', 'category_1', \n             'category_3', 'merchant_category_id', 'merchant_id', \n             'category_2', 'state_id', 'subsector_id']\n\nfor col in cat_cols:\n    cat_num = new_merchant_transactions_df[col].value_counts().index.shape[0]\n    print(f'{cat_num} unique values in {col}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a44cefce1d8b541d93f6e69588046b8d81ed17fb"},"cell_type":"code","source":"#example of continuous data on single card_id\nidx = np.random.choice(new_merchant_transactions_df.card_id)\ndf = new_merchant_transactions_df[new_merchant_transactions_df.card_id == idx]\nsns.pairplot(df[['month_lag', 'purchase_amount']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fd7f6979604f5e9dd1eb52a3614e000423328e4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}