{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Elo Merchant Category Recommendation\n## Simple EDA, feature engineering and LGB model\n\n#### thanks to all members"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Python ≥3.5 is required\nimport sys\nassert sys.version_info >= (3, 5)\n\n# Scikit-Learn ≥0.20 is required\nimport sklearn\nassert sklearn.__version__ >= \"0.20\"\n\n# Common imports\nimport numpy as np\nimport os\nimport gc\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\n\n%matplotlib inline \n#plotting directly without requering the plot()\n\nimport warnings\nwarnings.filterwarnings(action=\"ignore\") #ignoring most of warnings, cleaning up the notebook for better visualization\n\npd.set_option('display.max_columns', 500) #fixing the number of rows and columns to be displayed\npd.set_option('display.max_rows', 500)\n\nprint(os.listdir(\"../input\")) #showing all the files in the ../input directory\n\n# Any results you write to the current directory are saved as output. Kaggle message :D","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', parse_dates=['first_active_month'], low_memory=True)\ntest = pd.read_csv('../input/test.csv', low_memory=True, parse_dates=['first_active_month'])\ndf_new_merch = pd.read_csv('../input/new_merchant_transactions.csv', parse_dates=True, low_memory=True)\ndf_hist_trans = pd.read_csv('../input/historical_transactions.csv',parse_dates=True, low_memory=True)\n\nprint('Training set shape: {}'.format(train.shape))\nprint('Testing set shape: {}'.format(train.shape))\nprint('Merchants set shape: {}'.format(df_new_merch.shape))\nprint('Historical transactions set shape: {}'.format(df_hist_trans.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Max date for the training set: {}'.format(train['first_active_month'].max()))\nprint('Max date for the testing set: {}'.format(test['first_active_month'].max()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['year'] = train['first_active_month'].dt.year\ntrain['month'] = train['first_active_month'].dt.month\ntest['year'] = test['first_active_month'].dt.year\ntest['month'] = test['first_active_month'].dt.month\nplt.subplot(3,1,1)\ntrain['target'].plot.hist(bins=30, edgecolor='black', figsize=(16,10))\nplt.subplot(3,1,2)\nsns.countplot(train['year'])\nplt.subplot(3,1,3)\nsns.countplot(train['month'])\nplt.xticks(rotation='vertical')\nplt.title('Some title')\nplt.tight_layout(h_pad=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['target'] < - 30]['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (16, 12))\n\n\ntrain['log_target'] = np.log1p(train['target'])\n\n\n# iterate through the sources\nfor i, features in enumerate(['feature_1', 'feature_2', 'feature_3']):\n    \n    # create a new subplot for each source\n    plt.subplot(3, 1, i + 1)\n    # plot repaid loans\n    sns.kdeplot(train.loc[train['log_target'] >= 0, features], label = 'log_target >= 0')\n    # plot loans that were not repaid\n    sns.kdeplot(train.loc[train['log_target'] < 0, features], label = 'log_target < 0')\n    \n    # Label the plots\n    plt.title('Distribution of %s by log 1p target Value' % features)\n    plt.xlabel('%s' % features); plt.ylabel('Density');\n    \nplt.tight_layout(h_pad = 1.5)\n\ntrain.drop('log_target', axis=1, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_hist_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group_purch = df_hist_trans.groupby('card_id')['purchase_amount'].size().reset_index()\ngroup_purch.columns = [\"card_id\", \"count_purchase_amount\"]\n\ntrain = pd.merge(train,\n                 group_purch,\n                 on='card_id',\n                 how='left')\n\ntest = pd.merge(test,\n                 group_purch,\n                 on='card_id',\n                 how='left')\n\n\nprint('Training set shape after merging with historical transaction: {}'.format(train.shape))\n\nprint('Testing set shape after merging with historical transaction: {}'.format(test.shape))\n\ngc.enable\ndel df_hist_trans, group_purch\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16,5))\nplt.subplot(2,1,1)\nplt.scatter(train['count_purchase_amount'], train['target'], c='c')\nplt.subplot(2,1,2)\ntrain.set_index('first_active_month')['count_purchase_amount'].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_new_merch.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group_card_temp = df_new_merch.groupby('card_id')['purchase_amount'].size().reset_index()\n\ngroup_card_temp.columns = ['card_id', 'count_new_merch_purchases']\n\ntrain = pd.merge(train,\n                group_card_temp,\n                on='card_id',\n                how='left')\n\ntest = pd.merge(test,\n                group_card_temp,\n                on='card_id',\n                how='left')\n\nprint('Training set shape after merging with new merchant transaction: {}'.format(train.shape))\n\nprint('Testing set shape after merging with new merchant transaction: {}'.format(test.shape))\n\ntrain['count_new_merch_purchases'].fillna(0,inplace=True)\ntest['count_new_merch_purchases'].fillna(0,inplace=True)\n\n\ndel group_card_temp, df_new_merch\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16,5))\nplt.subplot(2,1,1)\nsns.scatterplot(x='count_purchase_amount', y='target', data= train)\nplt.title('Relationship between target value and purchase amount')\nplt.subplot(2,1,2)\nsns.scatterplot(x='count_new_merch_purchases', y='target', data= train)\nplt.title('Relationship between target value and new merchants purchase amount')\nplt.tight_layout(h_pad=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16,5))\nplt.subplot(2,1,1)\ntrain.set_index('first_active_month')['count_purchase_amount'].plot(c='b')\nplt.title('Purchase amount through time')\nplt.subplot(2,1,2)\ntrain.set_index('first_active_month')['count_new_merch_purchases'].plot(c='r')\nplt.title('New merchantes purchase amount through time')\nplt.tight_layout(h_pad=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16,8))\ncorr_train = train.corr()\nsns.heatmap(corr_train, cmap = plt.cm.RdYlBu_r, vmin = -0.25, annot = True, vmax = 0.6)\nplt.title('Correlation Heatmap');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = train['target'].copy()\nmask = ['target','first_active_month','card_id']\ncols = [col for col in train.columns if col not in mask]\ntrain_prepared = train.loc[:, cols]\ntest_prepared = test.loc[:, cols]\n\nprint('Training set: {}'.format(train_prepared.shape))\nprint('testing set: {}'.format(test_prepared.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" #let's use the Imputer to fill the NAN values with the median value\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler,MinMaxScaler,Imputer, RobustScaler\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\nimport time #implementing in this function the time spent on training the model\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV,cross_val_score,train_test_split, KFold\nimport lightgbm as lgb\nimport gc\n\n#imputing all NaN value\npipeline = Pipeline([\n        ('imputer', SimpleImputer(strategy=\"median\")), \n        #('scale', MinMaxScaler(feature_range = (0, 1))),\n        ('robustScaler', RobustScaler()),\n])\n\nnfolds = 10\nfolds = KFold(n_splits=nfolds, shuffle=True,random_state=42)\n\n\n#Generic function for making a classification model and accessing performance:\ndef fit_model(train, train_labels, test_set, params={}, \n                         fold=folds, model=None, \n                         GridSearch=False, plot_features_importances=False):\n    \n    time_start = time.perf_counter() #start counting the time\n    #creating our validation set out of the training set and labels provided\n    X_train, x_val, y_train, y_val = train_test_split(train, train_labels, test_size=0.1, random_state=42)\n    X_train = pipeline.fit_transform(X_train) #fiting and transforming the dataset using the pipeline provided\n    x_val = pipeline.fit_transform(x_val)\n    \n    test_sub = np.zeros(test_set.shape[0])\n    test_set = pipeline.fit_transform(test_set)\n    \n    predict_val = np.zeros(train.shape[0])\n    score = {}\n    \n    if model != None: grid_model = GridSearchCV(model, params,verbose=1, cv=3) #initializing the grid search model\n\n    if GridSearch:\n        grid_model.fit(X_train, y_train)\n        score_grid = grid_model.best_score_\n        \n        #predicting using the model that has been trained above\n        \n        predict_val = grid_model.predict(x_val)\n        score['MAE'] = mean_absolute_error(y_val, predict_val)\n        score['RMSE'] = np.sqrt(mean_squared_error(y_val, predict_val))\n        \n        print(\"Model Report\")\n\n        print(\"MAE: \"+ str(score[\"MAE\"]))\n        print(\"RMSE: \"+ str(score[\"RMSE\"]))\n        print('\\n')\n    \n        test_sub = grid_model.predict(test_set) \n        \n    else:\n        model = lgb.LGBMRegressor(**params, n_estimators = 5000, nthread = 4, n_jobs = -1)\n\n        for n, (index, val_index) in enumerate(folds.split(train)):\n            \n            print('Starting Fold number: %d' %n)\n            X, X_val = train.values[index], train.values[val_index]\n            Y, Y_val = train_labels[index], train_labels[val_index]\n            X = pipeline.fit_transform(X)\n            X_val = pipeline.fit_transform(X_val)\n            \n            model.fit(X, Y, \n                    eval_set=[(X, Y), (X_val, Y_val)],\n                    verbose=1000, early_stopping_rounds=200)\n            \n            predict_val = model.predict(X_val)\n            test_temp = model.predict(test_set, num_iteration=model.best_iteration_)\n            test_sub += test_temp\n            \n            if score == {}:\n                score['MAE'] = mean_absolute_error(Y_val, predict_val)\n                score['RMSE'] = np.sqrt(mean_squared_error(Y_val, predict_val))            \n            else:\n                score['MAE'] += mean_absolute_error(Y_val, predict_val)\n                score['RMSE'] += np.sqrt(mean_squared_error(Y_val, predict_val))\n\n                        \n        test_sub /= nfolds\n                        \n        print(\"Model Report\")\n\n        print(\"MAE: \"+ str(score[\"MAE\"]/nfolds))\n        print(\"RMSE: \"+ str(score[\"RMSE\"]/nfolds))\n        \n        print('\\n')\n\n        \n    #################### PLOTTING FEATURES IMPORTANCE ####################\n    \n    # Sort features according to importance\n    if plot_features_importances:\n        if GridSearch:\n            # Extract feature importances\n            feature_importances = pd.DataFrame({'feature': list(train.columns), 'importance': grid_model.best_estimator_.feature_importances_})\n        else:\n            feature_importances = pd.DataFrame({'feature': list(train.columns), 'importance': model.feature_importances_})\n        \n        feature_importances = feature_importances.sort_values('importance', ascending = False).reset_index()\n\n        # Normalize the feature importances to add up to one\n        feature_importances['importance_normalized'] = feature_importances['importance'] / feature_importances['importance'].sum()\n\n        # Make a horizontal bar chart of feature importances\n        plt.figure(figsize = (10, 6))\n        ax = plt.subplot()\n\n        # Need to reverse the index to plot most important on top\n        ax.barh(list(reversed(list(feature_importances.index[:15]))), \n                feature_importances['importance_normalized'].head(15), \n                align = 'center', edgecolor = 'k')\n\n        # Set the yticks and labels\n        ax.set_yticks(list(reversed(list(feature_importances.index[:15]))))\n        ax.set_yticklabels(feature_importances['feature'].head(15))\n\n        # Plot labeling\n        plt.xlabel('Normalized Importance'); plt.title('Feature Importances')\n    \n    time_end = time.perf_counter() #end of counting the time\n    \n    total_time = time_end-time_start #total time spent during training and cross_validation\n    \n    print(\"Amount of time spent during training the model and cross validation: %4.3f seconds\" % (total_time))\n    # Clean up memory\n    gc.enable()\n    del model, X_train, x_val, y_train, y_val,score, total_time, time_end, time_start,predict_val,test_set\n    gc.collect()\n                        \n    return test_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########## LGB ########\nparams = {\n          'num_leaves': 30,\n         'min_data_in_leaf': 20,\n         'objective': 'regression',\n         'max_depth': 10,\n         'learning_rate': 0.01,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.2,\n}\nprediction_lgb = fit_model(train_prepared,train_labels, test_prepared, params=params, plot_features_importances=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test[\"card_id\"].values})\nsub_df[\"target\"] = prediction_lgb\nsub_df.to_csv(\"lgbm.csv\", index=False)\nsub_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}