{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom datetime import date, datetime\nimport gc\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nfrom imblearn.pipeline import make_pipeline as make_pipeline_imb # To do our transformation in a unique time\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.pipeline import make_pipeline\nfrom imblearn.metrics import classification_report_imbalanced\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score, KFold, GridSearchCV\nfrom collections import Counter\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\n\nfrom sklearn.metrics import precision_score, recall_score, fbeta_score, confusion_matrix, precision_recall_curve, accuracy_score, mean_squared_error\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"## As is evident from multiple discussions: \n* https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73571\n* https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73922\n* https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73024\n\n## The 2207 rows with target values < -33 do seem to have an important role in this synthesized dataset. I wanted to do a little trial where I wanted to check what kind of accuracy the model was getting in predicting these outliers on the training set. So we have ourselves an imbalanced binary classification task, that I will balance out via SMOTE. \n\n## The longer term idea is that I want to use the model that predicts these outliers best (for the test set predictions)\n\n## Upvote the kernel if you like it, and please do point out any conceptual errors or flaws that you notice! \n"},{"metadata":{"_uuid":"2ca3b0f2b821cab790ae0eff663fcec0fc6b9ca6"},"cell_type":"markdown","source":"## The ideas for SMOTE come from https://www.kaggle.com/kabure/credit-card-fraud-prediction-rf-smote"},{"metadata":{"trusted":true,"_uuid":"01980f62e7c20aa01515285cfe2c095701a1cd68"},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')\ndf_hist_trans = pd.read_csv('../input/historical_transactions.csv')\ndf_new_merchant_trans = pd.read_csv('../input/new_merchant_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da7807fab097f097dbc86c08fd67ea04f4c5020c"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bcc2d8f5f92fa058d661ab40047bace2c34c84b"},"cell_type":"code","source":"for df in [df_hist_trans,df_new_merchant_trans]:\n    df['category_2'].fillna(1.0,inplace=True)\n    df['category_3'].fillna('A',inplace=True)\n    df['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d960cce083f8adb3ca6694634499561cb8a3c93e"},"cell_type":"code","source":"def get_new_columns(name,aggs):\n    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f86e7771e1c41c1c0b41f3d1c0a2e3fd0b3e140b"},"cell_type":"code","source":"for df in [df_hist_trans,df_new_merchant_trans]:\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n    df['category_1'] = df['category_1'].map({'Y':1, 'N':0}) \n    df['month_diff'] = ((datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c31345f3c830597bf616245223f4bee351af4c48"},"cell_type":"code","source":"aggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\n\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['authorized_flag'] = ['sum', 'mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    df_hist_trans[col+'_mean'] = df_hist_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']    \n\nnew_columns = get_new_columns('hist',aggs)\ndf_hist_trans_group = df_hist_trans.groupby('card_id').agg(aggs)\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\ndf_hist_trans_group['hist_purchase_date_diff'] = (df_hist_trans_group['hist_purchase_date_max'] - df_hist_trans_group['hist_purchase_date_min']).dt.days\ndf_hist_trans_group['hist_purchase_date_average'] = df_hist_trans_group['hist_purchase_date_diff']/df_hist_trans_group['hist_card_id_size']\ndf_hist_trans_group['hist_purchase_date_uptonow'] = (datetime.today() - df_hist_trans_group['hist_purchase_date_max']).dt.days\n\ndf_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\n\ndel df_hist_trans_group;\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e0fc47f87f3b77df77c82711d15e49847295125"},"cell_type":"code","source":"aggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    df_new_merchant_trans[col+'_mean'] = df_new_merchant_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']\n    \nnew_columns = get_new_columns('new_hist',aggs)\ndf_hist_trans_group = df_new_merchant_trans.groupby('card_id').agg(aggs)\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\ndf_hist_trans_group['new_hist_purchase_date_diff'] = (df_hist_trans_group['new_hist_purchase_date_max'] - df_hist_trans_group['new_hist_purchase_date_min']).dt.days\ndf_hist_trans_group['new_hist_purchase_date_average'] = df_hist_trans_group['new_hist_purchase_date_diff']/df_hist_trans_group['new_hist_card_id_size']\ndf_hist_trans_group['new_hist_purchase_date_uptonow'] = (datetime.today() - df_hist_trans_group['new_hist_purchase_date_max']).dt.days\n\ndf_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\n\ndel df_hist_trans_group;\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a99c4d152da2f573951d991cb665957f2f4b4ae"},"cell_type":"code","source":"del df_hist_trans;\ngc.collect()\n\ndel df_new_merchant_trans;\ngc.collect()\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cab84c550d9bc89d6f9f0a6de0c65c81ade961f"},"cell_type":"code","source":"df_train['first_active_month'] = pd.to_datetime(df_train['first_active_month'])\ndf_train['dayofweek'] = df_train['first_active_month'].dt.dayofweek\ndf_train['weekofyear'] = df_train['first_active_month'].dt.weekofyear\ndf_train['month'] = df_train['first_active_month'].dt.month\ndf_train['elapsed_time'] = (datetime.today() - df_train['first_active_month']).dt.days\ndf_train['hist_first_buy'] = (df_train['hist_purchase_date_min'] - df_train['first_active_month']).dt.days\ndf_train['new_hist_first_buy'] = (df_train['new_hist_purchase_date_min'] - df_train['first_active_month']).dt.days\nfor f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\\\n                    'new_hist_purchase_date_min']:\n    df_train[f] = df_train[f].astype(np.int64) * 1e-9\ndf_train['card_id_total'] = df_train['new_hist_card_id_size']+df_train['hist_card_id_size']\ndf_train['purchase_amount_total'] = df_train['new_hist_purchase_amount_sum'] + df_train['hist_purchase_amount_sum']\ndf_train = pd.get_dummies(df_train, columns=['feature_1', 'feature_2'])\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0914f47d67a62864709024b71508922b474d53b7"},"cell_type":"code","source":"df_train['outliers_target'] = 0\ndf_train.loc[df_train['target'] < -30, 'outliers_target'] = 1\ndf_train['outliers_target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e757649949650e4aba8704898a0ae09d8d323d4f"},"cell_type":"markdown","source":"### So if we consider the outliers to be the target variable (for binary classification), we have a rather imbalanced dataset (which we will then balance out via Smote)\n### Smote will likely have issues with nans, so remove all those columns for now. Later we may use subtler methods like sklearn imputer etc. "},{"metadata":{"trusted":true,"_uuid":"fba82a26bb020a71f71b8c1cd212e96b7f57c51d"},"cell_type":"code","source":"df_train.columns[df_train.isna().any()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f4b7f520906f22e67dbed12489de819d5684215"},"cell_type":"code","source":"df_train.drop(['new_hist_month_nunique', 'new_hist_hour_nunique',\n       'new_hist_weekofyear_nunique', 'new_hist_dayofweek_nunique',\n       'new_hist_year_nunique', 'new_hist_subsector_id_nunique',\n       'new_hist_merchant_id_nunique', 'new_hist_merchant_category_id_nunique',\n       'new_hist_purchase_amount_sum', 'new_hist_purchase_amount_max',\n       'new_hist_purchase_amount_min', 'new_hist_purchase_amount_mean',\n       'new_hist_purchase_amount_var', 'new_hist_installments_sum',\n       'new_hist_installments_max', 'new_hist_installments_min',\n       'new_hist_installments_mean', 'new_hist_installments_var',\n       'new_hist_month_lag_max', 'new_hist_month_lag_min',\n       'new_hist_month_lag_mean', 'new_hist_month_lag_var',\n       'new_hist_month_diff_mean', 'new_hist_weekend_sum',\n       'new_hist_weekend_mean', 'new_hist_category_1_sum',\n       'new_hist_category_1_mean', 'new_hist_card_id_size',\n       'new_hist_category_2_mean_mean', 'new_hist_category_3_mean_mean',\n       'new_hist_purchase_date_diff', 'new_hist_purchase_date_average',\n       'new_hist_purchase_date_uptonow', 'new_hist_first_buy', 'card_id_total',\n       'purchase_amount_total'], axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b10af72593c79104a787fe5ae5deb8f038e96059","scrolled":true},"cell_type":"code","source":"X = df_train.drop(['target','card_id','first_active_month','outliers_target'],axis=1)\ny = df_train['outliers_target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0db510a17b15f495a87b64b978ee0a735236d77d"},"cell_type":"code","source":"def print_results(headline, true_value, pred):\n    print(headline)\n    print(\"accuracy: {}\".format(accuracy_score(true_value, pred)))\n    print(\"precision: {}\".format(precision_score(true_value, pred)))\n    print(\"recall: {}\".format(recall_score(true_value, pred)))\n    print(\"f2: {}\".format(fbeta_score(true_value, pred, beta=2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28efcc230d9bd6c064a3501fba8fdeff880a4763"},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.20)\n\nclassifier = RandomForestClassifier\n\n# build model with SMOTE imblearn\nsmote_pipeline = make_pipeline_imb(SMOTE(random_state=42), \\\n                                   classifier(random_state=42))\n\nsmote_model = smote_pipeline.fit(X_train, y_train)\nsmote_prediction = smote_model.predict(X_test)\n\n\nprint(\"normal data distribution: {}\".format(Counter(y)))\n\nX_smote, y_smote = SMOTE().fit_sample(X, y)\n\nprint(\"SMOTE data distribution: {}\".format(Counter(y_smote)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"275e56c6c72789ca06f0ab5134c29ef18e6dacb5"},"cell_type":"code","source":"print(\"Confusion Matrix: \")\nprint(confusion_matrix(y_test, smote_prediction))\n\nprint('\\nSMOTE Pipeline Score {}'.format(smote_pipeline.score(X_test, y_test)))\n\nprint_results(\"\\nSMOTE + RandomForest classification\", y_test, smote_prediction)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3f7ec2e6928fb708267b08ba48de26ec25cf304"},"cell_type":"markdown","source":"### So we see quite a high number of False Negatives, which drags the Recall down, and implies that we aren't getting as many of the outliers as we should be. A lot of this can, of course, be improved by parameter tuning."},{"metadata":{"trusted":true,"_uuid":"79a4a9b6024401e31154517759132dfc0eb2c24a"},"cell_type":"markdown","source":"### Next steps:: \n#### * Tune the hyperparmeters to get a better Recall on the 'outlier_targets'\n#### * See how the performance of this classifier ends up affecting the performance of the actual model on the validation set \n\n### So this was my first effort at trying to account for the outliers in the training set. Any comments/clarifications/corrections are more than welcome! \n#### To be continued... "},{"metadata":{"trusted":true,"_uuid":"2417d32bd9a42ae5b1aeb64ca323804f4f8bef70"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}