{"cells":[{"metadata":{"_uuid":"ff663ba51ae5d49b17b96d24743c6059f8ca6637"},"cell_type":"markdown","source":"# User Loyalty prediction\nLet us explore this regression problem."},{"metadata":{"trusted":true,"_uuid":"86a04b03f3c24bcbe7fe1a54d59276902b8bcdea","_kg_hide-output":true,"_kg_hide-input":true,"scrolled":false},"cell_type":"code","source":"!pip install torchtext==0.2.3\n!pip install fastai==0.7.0\n\nif 1==2:\n    from fastai.imports import *\n    from fastai.transforms import *\n    from fastai.conv_learner import *\n    from fastai.model import *\n    from fastai.dataset import *\n    from fastai.sgdr import *\n    from fastai.plots import *\n    from fastai.column_data import *\nfrom fastai.structured import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3581bd3ea80c0ed1bdf95c0992d2609c6293e66e"},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom sklearn.linear_model import ElasticNet, Ridge, LinearRegression, Lasso\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import RobustScaler\nimport seaborn as sea\nimport gc\nimport matplotlib.style as style \n\nstyle.use('seaborn-notebook') #sets the size of the charts\nstyle.use(\"seaborn-pastel\")\n#style.use('ggplot')\n\nimport os\nprint(os.listdir(\"../input\"))\nPATH = \"../input/\"\nPATH_TMP = \"../../tmp/\"\nPATH_MODEL = \"../../model/\"\nblnforoutlier = 0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00001f7cef93fe78c290efb2803c1688e6531a1f"},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true,"_uuid":"5c5a696ba99c4060e8d164d0c58d52b888acc493"},"cell_type":"code","source":"def load_data():\n    df_train = pd.read_csv(f'{PATH}train.csv')\n    df_test = pd.read_csv(f'{PATH}test.csv')\n    df_train.head(1)\n    return df_train, df_test\ndef load_tran_data():\n    df_history = pd.read_csv(f'{PATH}historical_transactions.csv')\n    df_new = pd.read_csv(f'{PATH}new_merchant_transactions.csv')\n    df_mer = pd.read_csv(f'{PATH}merchants.csv')\n    df_history.head(1)\n    return df_history, df_new, df_mer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8a6e356cf8a5c3bf70f3b4d51cb5ac453f4b22c"},"cell_type":"markdown","source":"# Reduce memory usage.\nThis code is taken up from kernel by [Mitsuru](https://www.kaggle.com/mfjwr1)"},{"metadata":{"trusted":true,"_uuid":"658d346b7386a86ce3772a5ab371b14d7ff2b62b"},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ec07ca70c2a3ab2bc06620d55f24a59e2363064"},"cell_type":"markdown","source":"# Convert key cols to numeric as this leads to faster execution in grouping/merging operations"},{"metadata":{"trusted":true,"_uuid":"3e413c0f3ab14fa945f3088bc271189bfb05b9dd"},"cell_type":"code","source":"def convert_merchant_id():\n    global df_mer, df_history, df_new\n    dic_merchant_id = {v:k for k,v in enumerate(list(df_mer[\"merchant_id\"].unique()))}\n    df_mer[\"merchant_id_SNo\"] = df_mer[\"merchant_id\"].map(dic_merchant_id)\n    df_history[\"merchant_id_SNo\"] = df_history[\"merchant_id\"].map(dic_merchant_id)\n    df_new[\"merchant_id_SNo\"] = df_new[\"merchant_id\"].map(dic_merchant_id)\n    df_history.drop([\"merchant_id\"], axis=1,inplace=True)\n    df_new.drop([\"merchant_id\"], axis=1,inplace=True)\n    df_mer.drop([\"merchant_id\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"032a30097a636a993792890e27e28bf5e502cab1"},"cell_type":"code","source":"def convert_card_id():\n    global df_history, df_new, df_train, df_test\n    list_card_id = list(set(df_history[\"card_id\"].unique()) | set(df_new[\"card_id\"].unique()) | set(df_train[\"card_id\"].unique()) | set(df_test[\"card_id\"].unique()))\n    dic_card_id = {v:k for k,v in enumerate(list_card_id)}\n    df_new[\"card_id_SNo\"] = df_new[\"card_id\"].map(dic_card_id)\n    df_history[\"card_id_SNo\"] = df_history[\"card_id\"].map(dic_card_id)\n    df_train[\"card_id_SNo\"] = df_train[\"card_id\"].map(dic_card_id)\n    df_test[\"card_id_SNo\"] = df_test[\"card_id\"].map(dic_card_id)\n\n    df_history.drop('card_id',axis=1, inplace=True)\n    df_new.drop('card_id',axis=1, inplace=True)\n    df_train.drop('card_id',axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a432ddd9391871274b5a51ccc708b2026429af72"},"cell_type":"markdown","source":"# Data Cleaning\nMerchant data has duplicate records for merchant_id. We need to take care of duplicates. We can either take only one set which is the latest set or we can take mean of values in both the sets. We can assume duplicate data is there we got data of same merchant twice and we will just pick the latest one."},{"metadata":{"trusted":true,"_uuid":"a979fe5a4701bb5b4337fc5c6d4880fe846c4ed9"},"cell_type":"code","source":"def check_duplicates(in_df, key_col, blnDrop):\n    in_df[\"num_rec\"] = 0\n    df_grouped = in_df[[\"num_rec\",key_col]].groupby([key_col]).count()\n    df_grouped_duplicate = df_grouped[df_grouped[\"num_rec\"] > 1]\n    # Not sure what to do with these multiple recs. So will just take last index of each merchant_d\n    df_duplicate = in_df[in_df[key_col].isin(list(df_grouped_duplicate.index.values))]\n    print(\"No of duplicate records: \", df_duplicate.shape[0])\n    if blnDrop == True:\n        df_duplicate_max = in_df.loc[df_duplicate.index].reset_index()[[key_col,\"index\"]].groupby([key_col]).max()\n        drop_index = list(set(df_duplicate.index.values) - set(df_duplicate_max[\"index\"].values))\n        in_df.drop(drop_index, inplace=True)\n        del df_duplicate_max\n    del df_grouped, df_grouped_duplicate, df_duplicate\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d6bd9e3d3ffdb2041a4e0f5849db39d9497ffc6"},"cell_type":"markdown","source":"# Handle Null Values\n* For the single missing date, we can put the most common date\n* For category_3 we can put most frequent value\n* category_2 has many nulls. We should either drop the column or put mst frequent value\n* For category_2 in merchant we can put most frequent value\n* For lag columns we can put mean values\n* For merchants we will put the most popular merchant in matching criteria."},{"metadata":{"trusted":true,"_uuid":"031a9ff62cbe98cabc6263081192c8303e62ba3b"},"cell_type":"code","source":"def check_nulls(df, dfname):\n    allsum = df.shape[0]\n    for col in df.columns:\n        nasum = df[col].isna().sum()\n        if nasum > 0:\n            print(dfname, col, df[col].dtype, nasum, np.round((nasum*100)/allsum),2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68a3e35388b54403f66f3fee6f0d1e233031ad25"},"cell_type":"code","source":"def replace_null_with_most_freq_val(df, col):\n    df[\"temp_freq\"] = 0\n    most_freq_val = df[[col,\"temp_freq\"]].groupby([col], as_index=False).count().sort_values([\"temp_freq\"]).tail(1)[col].values[0]\n    null_col_index = list(df[df[col].isna()].index)\n    print(\"Most freq val\", most_freq_val, len(null_col_index))\n    df.loc[null_col_index,col] = most_freq_val\n    df.drop(\"temp_freq\",axis=1, inplace=True)\n    \ndef fix_missing_mer_for_group(in_df_mer, in_df_tran, group_cols):\n    in_df_mer.sort_values(\"num_his_rec\", inplace=True)\n    df_mer_grouped = in_df_mer[[\"merchant_id_SNo\"] + group_cols].groupby(group_cols, as_index=False).tail(1)\n\n    df_his_null = in_df_tran[in_df_tran[\"merchant_id_SNo\"].isna()]\n    print(\"Null Recs\", df_his_null.shape)\n    df_merged = pd.merge(df_his_null.reset_index()[group_cols + [\"index\"]] , df_mer_grouped, how=\"inner\", on=group_cols).set_index(\"index\")\n    print(\"Null recs rectified\", df_merged.shape)\n    in_df_tran.loc[list(df_merged.index), \"merchant_id_SNo\"] = df_merged.loc[list(df_merged.index), \"merchant_id_SNo\"]\n\n    df_his_null = in_df_tran[in_df_tran[\"merchant_id_SNo\"].isna()]\n    print(\"Null recs after fix\", df_his_null.shape)\n    \n    \ndef fix_missing_mer(in_df_tran, in_mer):\n    in_df_tran[\"num_rec\"] = 0\n    df_tran_grouped = in_df_tran[in_df_tran.isna() == False][[\"num_rec\",\"merchant_id_SNo\"]].groupby(\"merchant_id_SNo\").count()\n    \n    in_mer[\"num_his_rec\"] = 0\n    in_mer.set_index(\"merchant_id_SNo\").loc[list(df_tran_grouped.index), \"num_his_rec\"] = df_tran_grouped.loc[list(df_tran_grouped.index), \"num_rec\"]\n    in_mer[\"num_his_rec\"].fillna(0, inplace=True)\n\n    group_cols = [\"subsector_id\",\"merchant_category_id\",\"category_1\",\"category_2\",\"city_id\",\"state_id\"]\n    fix_missing_mer_for_group(in_mer, in_df_tran, group_cols)\n\n    group_cols = [\"subsector_id\",\"merchant_category_id\",\"category_1\",\"category_2\"] #,\"city_id\"] #,\"state_id\"]\n    fix_missing_mer_for_group(in_mer, in_df_tran, group_cols)\n\n    group_cols = [\"subsector_id\"] #,\"merchant_category_id\",\"category_1\",\"category_2\"] #,\"city_id\"] #,\"state_id\"]\n    fix_missing_mer_for_group(in_mer, in_df_tran, group_cols)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8c9d4de3cbd7d29c469fc39510aadd420328881"},"cell_type":"markdown","source":"# Explore categorical and continuous variables\nHow many values do they have, what is the distribution. Do we see any outliers etc."},{"metadata":{"trusted":true,"_uuid":"d4006ab2191b203f69b385c63a8a5db39da409cd"},"cell_type":"code","source":"def explore_label():\n    global df_train\n    df_train[\"outlier\"] = 0\n    df_train_outlier = df_train[(df_train[\"target\"] < -10) | (df_train[\"target\"] > 10)]\n    df_train_outlier = df_train_outlier.loc[df_train_outlier.index]\n    df_train.loc[df_train_outlier.index, \"outlier\"] = 1\n    df_train_without_outlier = df_train.drop(df_train_outlier.index)\n    \n    print(\"Percentage outlier:\", (df_train_outlier.shape[0]*100)/df_train.shape[0])\n    print(df_train.shape, df_train_outlier.shape, df_train_without_outlier.shape)\n    fig,ax = plt.subplots(nrows=1,ncols=3, figsize=(15,4))\n    df_train[[\"target\"]].plot(kind=\"hist\", bins=100, ax=ax[0], title=\"All data\");\n    df_train_outlier[\"target\"].plot(kind=\"hist\", bins=100, ax=ax[1], title=\"Outlier data\");\n    df_train_without_outlier[\"target\"].plot(kind=\"hist\", bins=100, ax=ax[2], title=\"Data Without Outier\");\n    del df_train_without_outlier\n    gc.collect()\n    if blnforoutlier == 1:\n        df_train[\"target\"] = df_train[\"outlier\"]\n    #df_train.drop([\"outlier\"], axis=1, inplace=True)\n    return df_train_outlier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d89a0ac0a4582ff851375437326255ca475f58e"},"cell_type":"code","source":"def explore_cat_cols(df):\n    global cat_cols\n    fig,ax = plt.subplots(nrows=2,ncols=5 , figsize=(20,8))\n    i = 0\n    j = 0\n    for col in cat_cols:\n        df[col].value_counts().plot(kind=\"bar\",ax=ax[i,j], title=col)\n        j = j + 1\n        if j == 5:\n            j=0\n            i = i + 1\n    plt.tight_layout()\n    \n    \ndef explore_cont_data():\n    global df_train\n    global df_train_outlier\n    fig,ax = plt.subplots(nrows=1,ncols=4, figsize=(20,4))\n    df_train[\"first_active_monthElapsed\"].plot(kind=\"hist\", bins=100, ax=ax[0], title=\"All Data\")\n    df_train_outlier[\"first_active_monthElapsed\"].plot(kind=\"hist\", bins=100, ax=ax[1], title=\"Outlier Data\")\n    df_train.sort_values([\"first_active_monthElapsed\"]).plot(kind=\"scatter\",x=\"first_active_monthElapsed\", y=\"target\", ax=ax[3], title=\"All Data\")\n    df_train_outlier.sort_values([\"first_active_monthElapsed\"]).plot(kind=\"scatter\",x=\"first_active_monthElapsed\", y=\"target\", ax=ax[2], title=\"Outlier Data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4dded615f1b249c37bece5bc77551829b98c25e"},"cell_type":"code","source":"def get_X_Y(feat_cols):\n    global df_train\n    global df_test\n    global label_col\n    scaler = StandardScaler()\n    X_all_raw = df_train[feat_cols].values.astype(np.float32)\n    y_all = df_train[label_col].values\n    X_test_raw = df_test[feat_cols].values.astype(np.float32)\n    outlier = df_train[\"outlier\"].values\n    \n    X_all = scaler.fit_transform(X_all_raw)\n    X_test = scaler.transform(X_test_raw)\n    X_train, X_valid, y_train, y_valid = train_test_split(X_all, y_all, stratify=outlier)\n    return X_train, X_valid, y_train, y_valid, X_test, X_all, y_all\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d0a025b35996a761ba12986442f71356808b4a4"},"cell_type":"markdown","source":"# Create various models  and predict for test and validation data\n* Microsoft Light GBM\n* fast.ai columnar data\n* SKLearn linear regression, ridge rgression, lasso regression, elastic regression\n* Tensorflow DNN\n* We will also see how stratified K Fold split helps us in improving results.\n* So let us code all of these."},{"metadata":{"trusted":true,"_uuid":"8880ad71562f6f2d0faf85ad6b49f51e0340e2b7"},"cell_type":"code","source":"def get_lgbm_pred():\n    global X_all, y_all, X_valid, y_valid, feature_cols, X_test\n    train_data = lgb.Dataset(X_all, label=y_all, free_raw_data=False)\n    test_data = lgb.Dataset(X_valid, label=y_valid,  free_raw_data=False)\n    params_1 = {\n            'task': 'train',\n            'boosting_type': 'gbdt',\n            'objective': \"regression\",\n            'verbose': 1,\n            'max_depth':7,\n            'num_leaves':70,\n            'learning_rate':0.01,\n             \n        }\n    if blnforoutlier == 1:\n        objective = \"binary\"\n    else:\n        objective = \"regression\"\n    params ={\n                'task': 'train',\n                'boosting': 'goss',\n                'objective': objective,\n                'metric': 'rmse',\n                'learning_rate': 0.0001,\n                'subsample': 0.9855232997390695,\n                'max_depth': 7,\n                'top_rate': 0.9064148448434349,\n                'num_leaves': 63,\n                'min_child_weight': 41.9612869171337,\n                'other_rate': 0.0721768246018207,\n                'reg_alpha': 9.677537745007898,\n                'colsample_bytree': 0.5665320670155495,\n                'min_split_gain': 9.820197773625843,\n                'reg_lambda': 8.2532317400459,\n                'min_data_in_leaf': 21,\n                'verbose': -1#,\n               # 'seed':int(2**n_fold),\n               # 'bagging_seed':int(2**n_fold),\n              #  'drop_seed':int(2**n_fold)\n                }\n\n    gbm_1 = lgb.train(params,\n            train_data,\n            valid_sets=test_data,\n            num_boost_round=50000,\n            early_stopping_rounds= 200,\n            feature_name=feature_cols,\n            categorical_feature='auto' #cat_cols\n            )\n\n    pred_valid = list(gbm_1.predict(X_valid))\n    valmse = mean_squared_error(pred_valid, y_valid)\n    print(valmse)\n    pred_test = list(gbm_1.predict(X_test))\n    return pred_test, valmse","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ac17a468ee304d62517eefd6c82c181386e4c15"},"cell_type":"code","source":"def get_keras_pred(num_epoch=20):\n    global X_all, y_all, X_test, X_valid, y_valid\n    from keras.models import Sequential\n    from keras.layers import Dense, Dropout\n    from keras.wrappers.scikit_learn import KerasRegressor\n    import keras\n\n    model = Sequential()\n    num_features = len(feature_cols)\n    model.add(Dropout(0.2, input_shape=(num_features,) ))\n    model.add(Dense(num_features*2,  kernel_initializer='normal', activation='relu'))\n    model.add(Dropout(0.2, input_shape=(num_features,) ))\n    model.add(Dense(num_features, kernel_initializer='normal', activation='relu'))\n    if blnforoutlier == 1:\n        model.add(Dense(1, kernel_initializer='normal'), activation=\"softmax\")\n        opt = keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n        model.compile(loss='categorical_crossentropy', optimizer=opt)\n    else:\n        model.add(Dense(1, kernel_initializer='normal'))\n        opt = keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n        model.compile(loss='mean_squared_error', optimizer=opt)\n    \n    model.fit(X_all, y_all, epochs=1000, batch_size=128)\n    scores = model.evaluate(X_all, y_all)\n    print(\"\\n%s: %.2f\" % (model.metrics_names[0], scores))\n    pred_valid = list(model.predict(X_valid))\n    valmse = mean_squared_error(pred_valid, y_valid)\n    print(valmse)\n    pred_test = model.predict(X_test)\n    return pred_test, valmse, model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87009ab7900b7e0260f9a71cc2c0a270f351d96c"},"cell_type":"code","source":"def get_sklearn_model(regr, X_train, X_valid, y_train, y_valid, X_test): \n    regr.fit(X_train, y_train)\n    y_pred = regr.predict(X_valid)\n    print(mean_squared_error(y_valid, y_pred))\n    return list(regr.predict(X_test))\n\n\ndef get_stratified_prediction(in_regr, in_X_all, in_y_all, in_outlier_all, in_X_test, in_feat_col):\n    folds = StratifiedKFold(n_splits=2, shuffle=True, random_state=2333)\n    y_pred = np.zeros(in_X_all.shape[0])\n    y_pred_test = np.zeros(in_X_test.shape[0])\n\n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(in_X_all,in_outlier_all)):\n        cur_X_train = in_X_all[trn_idx]\n        cur_y_train = in_y_all[trn_idx].astype(np.float64)\n        cur_X_val = in_X_all[val_idx]\n        in_regr.fit(cur_X_train, cur_y_train)\n        y_pred[val_idx] = in_regr.predict(cur_X_val)\n        cur_pred = in_regr.predict(in_X_test)\n        cur_pred = cur_pred/folds.n_splits\n        y_pred_test += cur_pred\n\n    print(mean_squared_error(in_y_all, y_pred))\n    feature_importance_df = pd.DataFrame({\"col\":in_feat_col, \"coef\":in_regr.coef_})\n    feature_importance_df.sort_values(\"coef\", inplace=True)\n    fig,ax = plt.subplots(nrows=1,ncols=2 , figsize=(20,8))\n    feature_importance_df.head(20).plot(kind=\"barh\",x=\"col\", y=\"coef\", ax=ax[0])\n    feature_importance_df.tail(20).plot(kind=\"barh\",x=\"col\", y=\"coef\", ax=ax[1])\n    plt.tight_layout()\n    return y_pred_test , feature_importance_df\n\n\ndef get_fast_ai_pred():\n    global df_train\n    global df_test\n    global cat_cols\n    df_train.reset_index(inplace=True)\n    all_idx = list(range(df_train.shape[0]))\n    train_idx, val_idx = train_test_split(all_idx)\n    for col in cat_cols:\n        df_train[col] = df_train[col].astype('category').cat.as_ordered()\n    apply_cats(df_test, df_train)\n    df, y, nas, mapper = proc_df(df_train[feature_cols+[\"target\"]], 'target', do_scale=True)\n    df_test.reset_index(inplace=True)\n    df_test[\"target\"] = 0.0\n    df_test_fi, _, nas, mapper = proc_df(df_test[feature_cols+[\"target\"]], 'target', do_scale=True,\n                                      mapper=mapper, na_dict=nas)\n    md = ColumnarModelData.from_data_frame(PATH, val_idx, df, y.astype(np.float32), cat_flds=cat_cols, bs=128,\n                                           test_df=df_test_fi)\n    categorical_col_data = [(c, len(df_train[c].cat.categories)+1) for c in cat_cols]\n    embd_sz = [(c, min(50, (c+1)//2)) for _,c in categorical_col_data]\n    m = md.get_learner(embd_sz, len(df.columns)-len(cat_cols),\n                   0.04, 1, [1000,500], [0.001,0.01], tmp_name=PATH_TMP, models_name=PATH_MODEL)\n    m.lr_find()\n    m.sched.plot_lr()\n    return m","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e0a04510669a626a23584f1efb9171845fd54c4"},"cell_type":"markdown","source":"# Feature Engineering \n* May be definition of loyalty could be predictability in behaviour so that we can know that our scheme has a high chance of being accepted.\n* So it could be like if a person always shops in he times of sale, and if sale was going on we cold let the person know.\n* Similarly if we know a person shops on a particula day\n* Also if a person shops from a particular brand or has a large number  of repeat brands\n* Also if a person shops at a particular size of store \n* Also if a person shops at a particula merchant category \n* Drop feaures wih high correlation from all data sets"},{"metadata":{"trusted":true,"_uuid":"2ea420cd341253423cfab8386fba3271532f5a17"},"cell_type":"code","source":"#for trend_col in [\"authorized_flag\", \"category_1\", \"city_id\"]: #,\"category_3\",\"merchant_category_id\",\"category_2\",\"state_id\",\"subsector_id\",\"merchant_id_SNo\"]:\n#for trend_col in [\"category_3\",\"merchant_category_id\",\"category_2\"]: #,\"state_id\",\"subsector_id\",\"merchant_id_SNo\"]:\ndef gen_feature_for_history_trend_col(trend_col_list, feature_cols,in_df,  prefix):\n    global df_test, df_train, key_col\n    org_feature_cols = feature_cols\n    for trend_col in trend_col_list:\n        print(trend_col)\n        dic_agg = {\"purchase_amount\": [\"sum\",\"count\"],\"installments\":[\"sum\"],\"month_lag\":[\"nunique\",\"max\"]}\n        df1 = in_df.groupby([key_col,trend_col]).agg(dic_agg)\n        df1.columns = [prefix + \"_\" + trend_col + \"_\" + col[0] + \"_\" + col[1] for col in df1.columns]\n        dic_agg2 = {}\n        for col in df1.columns:\n            dic_agg2[col] = [\"min\",\"max\",\"mean\",\"std\",\"skew\"]\n        df1.reset_index(inplace=True)\n        df1.set_index(key_col, inplace=True)\n        df2 = df1.groupby(df1.index).agg(dic_agg2)\n        df2.columns = [col[0] + \"_\" + col[1] for col in df2.columns]\n        tran_index = set(df2.index.values) & set(df_train.index.values)\n        for col in df2.columns:\n            df_train[col] = 0\n            df_train.loc[tran_index, col] = df2.loc[tran_index,col]\n        test_index = set(df2.index.values) & set(df_test.index.values)\n        for col in df2.columns:\n            df_test[col] = 0\n            df_test.loc[test_index, col] = df2.loc[test_index,col]\n        \n        all_cols = list(df_train.columns)\n        all_cols.remove(\"outlier\")\n        cr = df_train[all_cols].corr()\n        cr1 = cr[\"target\"]\n        feature_cols = list(cr1 [ (cr1>=0.03) | (cr1 <= -0.03)].index)\n        feature_cols.remove(\"target\")\n        for col in cr.columns:\n            crcol = cr[col]\n            highcrcol = list(crcol[crcol>0.65].index)\n            if col in highcrcol:\n                highcrcol.remove(col)\n                for col1 in highcrcol:\n                    if col1 != \"target\":\n                        cor1 = cr.loc[col,\"target\"]\n                        cor2 = cr.loc[col1,\"target\"]\n                        if cor1 < cor2:\n                            if col in feature_cols:\n                                feature_cols.remove(col)\n                        else:\n                            if col1 in feature_cols:\n                                feature_cols.remove(col1)\n                \n        for col in feature_cols:\n            df_train[col].loc[~np.isfinite(df_train[col])] = 0\n            df_test[col].loc[~np.isfinite(df_test[col])] = 0\n            df_train[col] = df_train[col].fillna(0)\n            df_test[col] = df_test[col].fillna(0) \n\n        cr1.sort_values()\n        for col in df_train[all_cols].columns:\n            if (col != \"target\") & (col not in feature_cols):\n                df_train.drop(col, axis=1, inplace=True)\n                if col in df_test.columns:\n                    df_test.drop(col, axis=1, inplace=True)\n    return feature_cols, list(set(feature_cols) - set(org_feature_cols))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8c7e1f7e918a020cf0866032ab4be4276e1b8f8"},"cell_type":"code","source":"df_train, df_test = load_data()\ndf_history, df_new, df_mer = load_tran_data()\n\nreduce_mem_usage(df_train)\nreduce_mem_usage(df_test)\nreduce_mem_usage(df_history)\nreduce_mem_usage(df_new)\nreduce_mem_usage(df_mer)\n\nconvert_merchant_id()\nconvert_card_id()\n\ncheck_duplicates(df_mer, \"merchant_id_SNo\", True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b122f153d16d6f5c01b7e5908df163606c7bed2"},"cell_type":"code","source":"print(df_history.shape)\nprint(df_mer.shape)\ncheck_nulls(df_train, \"train\")\ncheck_nulls(df_test, \"test\")\ncheck_nulls(df_history, \"history\")\ncheck_nulls(df_new, \"new\")\ncheck_nulls(df_mer, \"mer\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"469e1dfdf4c025fbb5137c194762de4073fc3333"},"cell_type":"code","source":"replace_null_with_most_freq_val(df_test, \"first_active_month\")\n\nreplace_null_with_most_freq_val(df_history, \"category_3\")\nreplace_null_with_most_freq_val(df_new, \"category_3\")\n\nreplace_null_with_most_freq_val(df_history, \"category_2\")\nreplace_null_with_most_freq_val(df_new, \"category_2\")\nreplace_null_with_most_freq_val(df_mer, \"category_2\")\n\nnull_lag3_index = list(df_mer[df_mer[\"avg_sales_lag3\"].isna()].index)\ndf_mer.loc[null_lag3_index,\"avg_sales_lag3\"] = df_mer[\"avg_sales_lag3\"].median()\n\nnull_lag6_index = list(df_mer[df_mer[\"avg_sales_lag6\"].isna()].index)\ndf_mer.loc[null_lag6_index,\"avg_sales_lag6\"] = df_mer[\"avg_sales_lag6\"].median()\n\nnull_lag12_index = list(df_mer[df_mer[\"avg_sales_lag12\"].isna()].index)\ndf_mer.loc[null_lag12_index,\"avg_sales_lag12\"] = df_mer[\"avg_sales_lag12\"].median()\n\n#fix_missing_mer(df_history, df_mer)\n#fix_missing_mer(df_new, df_mer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26fc04f49bb56b7cde65cbbc5bc5480e3b252998"},"cell_type":"code","source":"df_train_outlier = explore_label()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1d9edb73a9a41e2a8943f2b8166501d9174596a"},"cell_type":"code","source":"add_datepart(df_train, \"first_active_month\", drop=True)\nadd_datepart(df_test, \"first_active_month\", drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f20f395dc368ecf7653c8b01470acb8d96b4556"},"cell_type":"code","source":"label_col = \"target\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28c83142b6199c9f4d35ed589f80120ec871785e"},"cell_type":"code","source":"cr = df_train.corr()\ncr1 = cr[\"target\"]\nfeature_cols = list(cr1 [ (cr1>=0.03) | (cr1 <= -0.03)].index)\nfeature_cols.remove(\"target\")\nfeature_cols.remove(\"outlier\")\nfor col in feature_cols:\n    df_train[col] = df_train[col].fillna(0)\n    df_test[col] = df_test[col].fillna(0)\n\ncr1.sort_values()\nfor col in df_train.columns:\n    if (col != \"target\") & (col != \"outlier\") & (col != \"card_id_SNo\") & (col not in feature_cols):\n        df_train.drop(col, axis=1, inplace=True)\n        if col in df_test.columns:\n            df_test.drop(col, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6e02596f8f462901c532518d7f86237afba5601"},"cell_type":"code","source":"feature_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a270b9190085b74501f243b6ca8ab1ca312e32f"},"cell_type":"code","source":"key_col = \"card_id_SNo\"\ndf_train.set_index(key_col, inplace=True)\ndf_test.set_index(key_col, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"225087db3020d6171b7bee569ce33eb346fa12cf"},"cell_type":"code","source":"dflist = [df_history, df_new]\nprefix_list = [\"his\",\"new\"]\nfor i in range(len(dflist)):\n    in_df = dflist[i]\n    prefix = prefix_list[i]\n    feature_cols, new_features = gen_feature_for_history_trend_col([\"authorized_flag\"], feature_cols, in_df, prefix)\n    if 1==1:\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"state_id\",\"subsector_id\",\"merchant_id_SNo\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"category_1\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"city_id\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"category_3\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"merchant_category_id\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"category_2\"], feature_cols, in_df, prefix)\n\n        in_df.drop([\"state_id\",\"subsector_id\",\"authorized_flag\",\"category_1\",\"city_id\",\"category_3\",\"merchant_category_id\",\"category_2\"], axis=1, inplace=True)\n        add_datepart(in_df, \"purchase_date\",drop=True)\n\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Month\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Week\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Day\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Dayofweek\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Dayofyear\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Is_month_end\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Is_month_start\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Is_quarter_end\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Is_quarter_start\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Is_year_end\"], feature_cols, in_df, prefix)\n        feature_cols, new_features = gen_feature_for_history_trend_col([\"purchase_Is_year_start\"], feature_cols, in_df, prefix)\n\n        in_df.drop([\"purchase_Month\",\"purchase_Week\",\"purchase_Day\",\"purchase_Dayofweek\",\"purchase_Dayofyear\",\n                    \"purchase_Is_month_end\",\"purchase_Is_month_start\",\"purchase_Is_quarter_end\",\"purchase_Is_quarter_start\",\n                     \"purchase_Is_year_end\",\"purchase_Is_year_start\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2cdac707dc0018cfebb0f1559799445a196b654"},"cell_type":"code","source":"df_his_mer = pd.merge(df_history[[\"card_id_SNo\",\"merchant_id_SNo\",\"purchase_amount\",\"installments\",\"month_lag\"]], df_mer, how=\"inner\", on=\"merchant_id_SNo\")\ncol_list = df_his_mer.columns\ncol_list = list(set(col_list) - set([\"card_id_SNo\",\"merchant_id_SNo\",\"purchase_amount\",\"installments\",\"month_lag\"]))\nfor col in col_list:\n    numval = len(list(df_his_mer[col].unique()))\n    if numval <= 10:\n        feature_cols, new_features = gen_feature_for_history_trend_col([col], feature_cols, df_his_mer, \"his_mer\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1cc71de4078cbf98e65ea1efcd9808e6fc74e35"},"cell_type":"code","source":"df_new_mer = pd.merge(df_new[[\"card_id_SNo\",\"merchant_id_SNo\",\"purchase_amount\",\"installments\",\"month_lag\"]], df_mer, how=\"inner\", on=\"merchant_id_SNo\")\ncol_list = df_new_mer.columns\ncol_list = list(set(col_list) - set([\"card_id_SNo\",\"merchant_id_SNo\",\"purchase_amount\",\"installments\",\"month_lag\"]))\nfor col in col_list:\n    numval = len(list(df_his_mer[col].unique()))\n    if numval <= 10:\n        feature_cols, new_features = gen_feature_for_history_trend_col([col], feature_cols, df_new_mer, \"new_mer\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82b2e9a35b966d65f9e525301892b06fece69aa4"},"cell_type":"code","source":"X_train, X_valid, y_train, y_valid, X_test, X_all, y_all = get_X_Y(feature_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10b57110d82325ace2dab3687f9ae9591ec7aa66","_kg_hide-output":true},"cell_type":"code","source":"pred_test2, valmse, model = get_keras_pred()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b02df5dcff0482fb954ce042fbc5bfb788d17f9"},"cell_type":"code","source":"print(\"Loss\", valmse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43deb6f598eb69e337785b146f88a87fdc062ce4","_kg_hide-output":true},"cell_type":"code","source":"pred_test1, valmse = get_lgbm_pred()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"764e1dd6c0ef032815454f6b0fd60c4553e570b8"},"cell_type":"code","source":"print(\"Loss\", valmse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c04a12a7637ae9aa79b40ae31fcae7307e3e0113"},"cell_type":"code","source":"fig,ax = plt.subplots(figsize=(12,12))\nax = sns.heatmap(df_train[[\"target\"]  + feature_cols].corr(), ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"349f38d257bce41b5b26a52a4cd2a57d2b4ddd7b"},"cell_type":"code","source":"if 1==2:\n    regr = LinearRegression()\n    pred_test = get_sklearn_model(regr, X_train, X_valid, y_train, y_valid, X_test)\n    #This scores 3.924","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"974c0f57cd2c31613250a86439abe015ccbb7ea3"},"cell_type":"code","source":"if 1==2:\n    regr1 = LinearRegression()\n    pred_test, feature_df = get_stratified_prediction(regr1, X_all_dummy, y_all_dummy, outlier_all_dummy, X_test_dummy, feature_cols_with_dummy)\n    #This also scores 3.924","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04503d064fa4344dcfe68c37217a5eec63598633"},"cell_type":"code","source":"if 1==2:\n    regr = Lasso(alpha=0.005, max_iter=1000)\n    pred_test, feature_df = get_stratified_prediction(regr, X_all_dummy, y_all_dummy, outlier_all_dummy, X_test_dummy, feature_cols_with_dummy)\n    #This also scores 3.924","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"135046e2c8835936bb2c7f9744f901502f605a90"},"cell_type":"code","source":"if 1==2: #fast.ai is giving error\n    m = get_fast_ai_pred()\n    m.sched.plot()\n    plt.tight_layout()\n    plt.axvline(x=1.8e-2, color=\"red\");\n    m.fit(1e-2, 3) #, cycle_len=1, cycle_mult=2)\n    pred_test=m.predict(True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4e20ce6a5bf48c6fb21dacb4f832e99486fce01"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80835cfd0cd65604b1d104f88c21d42046d1c6d2"},"cell_type":"code","source":"import eli5\nfrom eli5.sklearn import PermutationImportance\nperm = PermutationImportance(model, random_state=1, scoring=\"neg_mean_squared_error\").fit(X_train,y_train)\neli5.show_weights(perm, feature_names = feature_cols, top=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c347a6a9269326d83ec4d324aff638cb163c6656"},"cell_type":"code","source":"#print(\"Loss\", valmse)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"bef30f4f409331ffa38a2c7533a35a8cce3033af"},"cell_type":"code","source":"df_test[\"target\"] = list(pred_test1)\ndf_test[[\"card_id\",\"target\"]].to_csv(\"submission1.csv\", index=False)\nfrom IPython.display import FileLink\nFileLink('submission1.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df0705ea20aaa5ea49566d8c9e3305d215afcd98"},"cell_type":"code","source":"df_test[\"target\"] = list(pred_test2.flatten())\ndf_test[[\"card_id\",\"target\"]].to_csv(\"submission2.csv\", index=False)\nfrom IPython.display import FileLink\nFileLink('submission1.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86a191d8e3edb7e29e514c838359e7af4bf1c3f2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3eddf87d046ded29a1bee4189785db16a3fc8796"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44bb9d2960dcb81e16af1b919c3aad68b6897e4d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87363f3baf74ca1ab4b5c3b7c2999226ee6f7070"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dabb91ce6872f493e3dec750e472d4acf8e7ae9e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"785b502b6db5e93a2aeedef4344534456269aabf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf6f68d498cdc968ae70c041bbd6bb0408a26600"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3add3f5c522a9f724051eebe58eefa3bca111b0e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}