{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport time\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import log_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3143b13f6cc576b2ab4b0ec46a6a648676d9a14"},"cell_type":"code","source":"import logging\nimport os\nimport gc\nimport time\nfrom datetime import datetime as dt\n\nimport numpy as np\nimport pandas as pd\nfrom pandas.core.common import SettingWithCopyWarning\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=SettingWithCopyWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\nimport lightgbm as lgb\nimport xgboost as xgb\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom contextlib import contextmanager\n\nlogger = logging.getLogger(__name__)\nlogging.basicConfig(\n    format='[%(levelname)s] %(asctime)s %(filename)s: %(lineno)d: %(message)s',\n    datefmt='%Y-%m-%d:%H:%M:%S',\n    level=logging.DEBUG)\n\nDATE_TODAY = dt(2019, 1, 26)\n\nFEATS_EXCLUDED = [\n    'first_active_month', 'target', 'card_id', 'outliers',\n    'hist_purchase_date_max', 'hist_purchase_date_min', 'hist_card_id_size',\n    'new_purchase_date_max', 'new_purchase_date_min', 'new_card_id_size',\n    'OOF_PRED', 'month_0']\n\n\n@contextmanager\ndef timer(title):\n    t0 = time.time()\n    yield\n    logger.info(\"{} - done in {:.0f}s\".format(title, time.time() - t0))\n\n\n# Display/plot feature importance\ndef display_importances(feature_importance_df_):\n    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n        by=\"importance\", ascending=False)[:40].index\n    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n\n    plt.figure(figsize=(8, 10))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n    plt.title('LightGBM Features (avg over folds)')\n    plt.tight_layout()\n    plt.savefig('lgbm_importances.png')\n\n\n# reduce memory\ndef reduce_mem_usage(df, verbose=True):\n    start_mem = df.memory_usage().sum() / 1024**2\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n                    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    return df\n\n\n# rmse\ndef rmse(y_true, y_pred):\n    return np.sqrt(mean_squared_error(y_true, y_pred))\n\n\n# One-hot encoding for categorical columns with get_dummies\ndef one_hot_encoder(df, nan_as_category=True):\n    original_columns = df.columns.tolist()\n\n    categorical_columns = list(filter(lambda c: c in ['object'], df.dtypes))\n    df = pd.get_dummies(df, columns=categorical_columns, dummy_na=nan_as_category)\n\n    new_columns = list(filter(lambda c: c not in original_columns, df.columns))\n    return df, new_columns\n\n\ndef process_main_df(df):\n    \n    # datetime features\n    df['quarter'] = df['first_active_month'].dt.quarter\n    df['elapsed_time'] = (DATE_TODAY - df['first_active_month']).dt.days\n\n    feature_cols = ['feature_1', 'feature_2', 'feature_3']\n    for f in feature_cols:    \n        df['days_' + f] = df['elapsed_time'] * df[f]\n        df['days_' + f + '_ratio'] = df[f] / df['elapsed_time']\n\n    # one hot encoding\n    df, cols = one_hot_encoder(df, nan_as_category=False)\n\n    df_feats = df.reindex(columns=feature_cols)\n    df['features_sum'] = df_feats.sum(axis=1)\n    df['features_mean'] = df_feats.mean(axis=1)\n    df['features_max'] = df_feats.max(axis=1)\n    df['features_min'] = df_feats.min(axis=1)\n    df['features_var'] = df_feats.std(axis=1)\n    df['features_prod'] = df_feats.product(axis=1)\n\n    return df\n\n\n# preprocessing train & test\ndef train_test(num_rows=None):\n\n    def read_csv(filename):\n        df = pd.read_csv(\n            filename, index_col=['card_id'], parse_dates=['first_active_month'], nrows=num_rows)\n        return df\n    \n    # load csv\n    train_df = read_csv('../input/train.csv')\n    test_df = read_csv('../input/test.csv') \n    logger.info(\"samples: train {}, test: {}\".format(train_df.shape, test_df.shape))\n\n    # outlier\n    train_df['outliers'] = 0\n    train_df.loc[train_df['target'] < -30., 'outliers'] = 1\n\n    train_df = reduce_mem_usage(process_main_df(train_df))\n    test_df = reduce_mem_usage(process_main_df(test_df))\n\n    feature_cols = ['feature_1', 'feature_2', 'feature_3']\n    for f in feature_cols:\n        order_label = train_df.groupby([f])['outliers'].mean()\n        train_df[f] = train_df[f].map(order_label)\n        test_df[f] = test_df[f].map(order_label)    \n\n    return train_df, test_df\n\n\ndef process_date(df):\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['month'] = df['purchase_date'].dt.month\n    df['day'] = df['purchase_date'].dt.day\n    df['hour'] = df['purchase_date'].dt.hour\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['weekday'] = df['purchase_date'].dt.weekday\n    df['weekend'] = (df['purchase_date'].dt.weekday >= 5).astype(int)\n    return df\n\n\ndef dist_holiday(df, col_name, date_holiday, date_ref, period=100):\n    df[col_name] = np.maximum(np.minimum((pd.to_datetime(date_holiday) - df[date_ref]).dt.days, period), 0)\n\n\ndef historical_transactions(num_rows=None):\n    \"\"\"\n    preprocessing historical transactions\n    \"\"\"\n    na_dict = {\n        'category_2': 1.,\n        'category_3': 'A',\n        'merchant_id': 'M_ID_00a6ca8a8a',\n    }\n\n    holidays = [\n        ('Christmas_Day_2017', '2017-12-25'),  # Christmas: December 25 2017\n        ('Mothers_Day_2017', '2017-06-04'),  # Mothers Day: May 14 2017\n        ('fathers_day_2017', '2017-08-13'),  # fathers day: August 13 2017\n        ('Children_day_2017', '2017-10-12'),  # Childrens day: October 12 2017\n        ('Valentine_Day_2017', '2017-06-12'),  # Valentine's Day : 12th June, 2017\n        ('Black_Friday_2017', '2017-11-24'),  # Black Friday: 24th November 2017\n        ('Mothers_Day_2018', '2018-05-13'),\n    ]\n\n    # agg\n    aggs = dict()\n    col_unique = ['subsector_id', 'merchant_id', 'merchant_category_id']\n    aggs.update({col: ['nunique'] for col in col_unique})\n\n    col_seas = ['month', 'hour', 'weekofyear', 'weekday', 'day']\n    aggs.update({col: ['nunique', 'mean', 'min', 'max'] for col in col_seas})\n\n    aggs_specific = {\n        'purchase_amount': ['sum', 'max', 'min', 'mean', 'var', 'skew'],\n        'installments': ['sum', 'max', 'mean', 'var', 'skew'],\n        'purchase_date': ['max', 'min'],\n        'month_lag': ['max', 'min', 'mean', 'var', 'skew'],\n        'month_diff': ['max', 'min', 'mean', 'var', 'skew'],\n        'authorized_flag': ['mean'],\n        'weekend': ['mean'], # overwrite\n        'weekday': ['mean'], # overwrite\n        'day': ['nunique', 'mean', 'min'], # overwrite\n        'category_1': ['mean'],\n        'category_2': ['mean'],\n        'category_3': ['mean'],\n        'card_id': ['size', 'count'],\n        'price': ['sum', 'mean', 'max', 'min', 'var'],\n        'Christmas_Day_2017': ['mean', 'sum'],\n        'Mothers_Day_2017': ['mean', 'sum'],\n        'fathers_day_2017': ['mean', 'sum'],\n        'Children_day_2017': ['mean', 'sum'],\n        'Valentine_Day_2017': ['mean', 'sum'],\n        'Black_Friday_2017': ['mean', 'sum'],\n        'Mothers_Day_2018': ['mean', 'sum'],\n        'duration': ['mean', 'min', 'max', 'var', 'skew'],\n        'amount_month_ratio': ['mean', 'min', 'max', 'var', 'skew'],\n    }\n    aggs.update(aggs_specific)\n\n    # starting to process\n    # load csv\n    df = pd.read_csv('../input/historical_transactions.csv', nrows=num_rows)\n    logger.info('read historical_transactions {}'.format(df.shape))\n    \n    # fillna\n    df.fillna(na_dict, inplace=True)\n    df['installments'].replace({\n        -1: np.nan, 999: np.nan}, inplace=True)\n\n    # trim\n    df['purchase_amount'] = df['purchase_amount'].apply(lambda x: min(x, 0.8))\n\n    # Y/N to 1/0\n    df['authorized_flag'] = df['authorized_flag'].map({'Y': 1, 'N': 0}).astype(np.int16)\n    df['category_1'] = df['category_1'].map({'Y': 1, 'N': 0}).astype(np.int16)\n    df['category_3'] = df['category_3'].map({'A': 0, 'B': 1, 'C':2}).astype(np.int16)\n\n    # additional features\n    df['price'] = df['purchase_amount'] / df['installments']\n\n    # datetime features\n    df = process_date(df)\n\n    # holidays\n    for d_name, d_day in holidays:\n        dist_holiday(df, d_name, d_day, 'purchase_date')\n\n    df['month_diff'] = (DATE_TODAY - df['purchase_date']).dt.days // 30\n    df['month_diff'] += df['month_lag']\n\n    # additional features\n    df['duration'] = df['purchase_amount'] * df['month_diff']\n    df['amount_month_ratio'] = df['purchase_amount'] / df['month_diff']\n\n    # reduce memory usage\n    df = reduce_mem_usage(df)\n\n    for col in ['category_2', 'category_3']:\n        df[col + '_mean'] = df.groupby([col])['purchase_amount'].transform('mean')\n        df[col + '_min'] = df.groupby([col])['purchase_amount'].transform('min')\n        df[col + '_max'] = df.groupby([col])['purchase_amount'].transform('max')\n        df[col + '_sum'] = df.groupby([col])['purchase_amount'].transform('sum')\n        aggs[col + '_mean'] = ['mean']\n    \n    df = df.reset_index().groupby('card_id').agg(aggs)\n\n    # change column name\n    df.columns = pd.Index([e[0] + \"_\" + e[1] for e in df.columns.tolist()])\n    df.columns = ['hist_' + c for c in df.columns]\n\n    df['hist_CLV'] = df['hist_card_id_count'] * df['hist_purchase_amount_sum'] / df['hist_month_diff_mean']\n\n    df['hist_purchase_date_diff'] = (df['hist_purchase_date_max'] - df['hist_purchase_date_min']).dt.days\n    df['hist_purchase_date_average'] = df['hist_purchase_date_diff'] / df['hist_card_id_size']\n    df['hist_purchase_date_uptonow'] = (DATE_TODAY - df['hist_purchase_date_max']).dt.days\n    df['hist_purchase_date_uptomin'] = (DATE_TODAY - df['hist_purchase_date_min']).dt.days\n\n    # reduce memory usage\n    df = reduce_mem_usage(df)\n\n    return df\n\n\ndef new_merchant_transactions(num_rows=None):\n    \"\"\"\n    preprocessing new_merchant_transactions\n    \"\"\"\n    na_dict = {\n        'category_2': 1.,\n        'category_3': 'A',\n        'merchant_id': 'M_ID_00a6ca8a8a',\n    }\n\n    holidays = [\n        ('Christmas_Day_2017', '2017-12-25'),  # Christmas: December 25 2017\n        # ('Mothers_Day_2017', '2017-06-04'),  # Mothers Day: May 14 2017\n        # ('fathers_day_2017', '2017-08-13'),  # fathers day: August 13 2017\n        ('Children_day_2017', '2017-10-12'),  # Childrens day: October 12 2017\n        # ('Valentine_Day_2017', '2017-06-12'),  # Valentine's Day : 12th June, 2017\n        ('Black_Friday_2017', '2017-11-24'),  # Black Friday: 24th November 2017\n        ('Mothers_Day_2018', '2018-05-13'),\n    ]\n    \n    aggs = dict()\n    col_unique = ['subsector_id', 'merchant_id', 'merchant_category_id']\n    aggs.update({col: ['nunique'] for col in col_unique})\n\n    col_seas = ['month', 'hour', 'weekofyear', 'weekday', 'day']\n    aggs.update({col: ['nunique', 'mean', 'min', 'max'] for col in col_seas})\n\n    aggs_specific = {\n        'purchase_amount': ['sum', 'max', 'min', 'mean', 'var', 'skew'],\n        'installments': ['sum', 'max', 'mean', 'var', 'skew'],\n        'purchase_date': ['max', 'min'],\n        'month_lag': ['max', 'min', 'mean', 'var', 'skew'],\n        'month_diff': ['mean', 'var', 'skew'],\n        'weekend': ['mean'],\n        'month': ['mean', 'min', 'max'],\n        'weekday': ['mean', 'min', 'max'],\n        'category_1': ['mean'],\n        'category_2': ['mean'],\n        'category_3': ['mean'],\n        'card_id': ['size', 'count'],\n        'price': ['mean', 'max', 'min', 'var'],\n        'Christmas_Day_2017': ['mean', 'sum'],\n        'Children_day_2017': ['mean', 'sum'],\n        'Black_Friday_2017': ['mean', 'sum'],\n        'Mothers_Day_2018': ['mean', 'sum'],\n        'duration': ['mean', 'min', 'max', 'var', 'skew'],\n        'amount_month_ratio': ['mean', 'min', 'max', 'var', 'skew'],\n    }\n    aggs.update(aggs_specific)\n\n    # load csv\n    df = pd.read_csv('../input/new_merchant_transactions.csv', nrows=num_rows)\n    logger.info('read new_merchant_transactions {}'.format(df.shape))\n    \n    # fillna\n    df.fillna(na_dict, inplace=True)\n    df['installments'].replace({\n        -1: np.nan, 999: np.nan}, inplace=True)\n\n    # trim\n    df['purchase_amount'] = df['purchase_amount'].apply(lambda x: min(x, 0.8))\n\n    # Y/N to 1/0\n    df['authorized_flag'] = df['authorized_flag'].map({'Y': 1, 'N': 0}).astype(int).astype(np.int16)\n    df['category_1'] = df['category_1'].map({'Y': 1, 'N': 0}).astype(int).astype(np.int16)\n    df['category_3'] = df['category_3'].map({'A': 0, 'B': 1, 'C': 2}).astype(int).astype(np.int16)\n\n    # additional features\n    df['price'] = df['purchase_amount'] / df['installments']\n\n    # datetime features\n    df = process_date(df)\n    for d_name, d_day in holidays:\n        dist_holiday(df, d_name, d_day, 'purchase_date')\n\n    df['month_diff'] = (DATE_TODAY - df['purchase_date']).dt.days // 30\n    df['month_diff'] += df['month_lag']\n\n    # additional features\n    df['duration'] = df['purchase_amount'] * df['month_diff']\n    df['amount_month_ratio'] = df['purchase_amount'] / df['month_diff']\n\n    # reduce memory usage\n    df = reduce_mem_usage(df)\n\n    for col in ['category_2', 'category_3']:\n        df[col+'_mean'] = df.groupby([col])['purchase_amount'].transform('mean')\n        df[col+'_min'] = df.groupby([col])['purchase_amount'].transform('min')\n        df[col+'_max'] = df.groupby([col])['purchase_amount'].transform('max')\n        df[col+'_sum'] = df.groupby([col])['purchase_amount'].transform('sum')\n        aggs[col + '_mean'] = ['mean']\n\n    df = df.reset_index().groupby('card_id').agg(aggs)\n\n    # change column name\n    df.columns = pd.Index([e[0] + \"_\" + e[1] for e in df.columns.tolist()])\n    df.columns = ['new_' + c for c in df.columns]\n\n    df['new_CLV'] = df['new_card_id_count'] * df['new_purchase_amount_sum'] / df['new_month_diff_mean']\n    \n    df['new_purchase_date_diff'] = (df['new_purchase_date_max'] - df['new_purchase_date_min']).dt.days\n    df['new_purchase_date_average'] = df['new_purchase_date_diff'] / df['new_card_id_size']\n    df['new_purchase_date_uptonow'] = (DATE_TODAY - df['new_purchase_date_max']).dt.days\n    df['new_purchase_date_uptomin'] = (DATE_TODAY - df['new_purchase_date_min']).dt.days\n\n    # reduce memory usage\n    df = reduce_mem_usage(df)\n\n    return df\n        \n\n# additional features\ndef additional_features(df):\n    \n    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days\n    df['hist_last_buy'] = (df['hist_purchase_date_max'] - df['first_active_month']).dt.days\n\n    df['new_first_buy'] = (df['new_purchase_date_min'] - df['first_active_month']).dt.days\n    df['new_last_buy'] = (df['new_purchase_date_max'] - df['first_active_month']).dt.days\n\n    date_features = [\n        'hist_purchase_date_max', 'hist_purchase_date_min', 'new_purchase_date_max', 'new_purchase_date_min']\n    for f in date_features:\n        df[f] = df[f].astype(np.int64) * 1e-9\n\n    #\n    df['card_id_total'] = df['new_card_id_size'] + df['hist_card_id_size']\n    df['card_id_cnt_total'] = df['new_card_id_count'] + df['hist_card_id_count']\n    df['card_id_cnt_ratio'] = df['new_card_id_count'] / df['hist_card_id_count']\n    \n    df['purchase_amount_total'] = df['new_purchase_amount_sum'] + df['hist_purchase_amount_sum']\n    df['purchase_amount_mean'] = df['new_purchase_amount_mean'] + df['hist_purchase_amount_mean']\n    df['purchase_amount_max'] = df['new_purchase_amount_max'] + df['hist_purchase_amount_max']\n    df['purchase_amount_min'] = df['new_purchase_amount_min'] + df['hist_purchase_amount_min']\n    df['purchase_amount_ratio'] = df['new_purchase_amount_sum'] / df['hist_purchase_amount_sum']\n\n    df['installments_total'] = df['new_installments_sum'] + df['hist_installments_sum']\n    df['installments_mean'] = df['new_installments_mean'] + df['hist_installments_mean']\n    df['installments_max'] = df['new_installments_max'] + df['hist_installments_max']\n    df['installments_ratio'] = df['new_installments_sum'] / df['hist_installments_sum']\n\n    df['price_total'] = df['purchase_amount_total'] / df['installments_total']\n    df['price_mean'] = df['purchase_amount_mean'] / df['installments_mean']\n    df['price_max'] = df['purchase_amount_max'] / df['installments_max']\n\n    #\n    df['month_diff_mean'] = df['new_month_diff_mean'] + df['hist_month_diff_mean']\n    df['month_diff_ratio'] = df['new_month_diff_mean'] / df['hist_month_diff_mean']\n    \n    df['month_lag_mean'] = df['new_month_lag_mean'] + df['hist_month_lag_mean']\n    df['month_lag_max'] = df['new_month_lag_max'] + df['hist_month_lag_max']\n    df['month_lag_min'] = df['new_month_lag_min'] + df['hist_month_lag_min']\n    df['category_1_mean'] = df['new_category_1_mean'] + df['hist_category_1_mean']\n        \n    df['duration_mean'] = df['new_duration_mean'] + df['hist_duration_mean']\n    df['duration_min'] = df['new_duration_min'] + df['hist_duration_min']\n    df['duration_max'] = df['new_duration_max'] + df['hist_duration_max']\n    \n    df['amount_month_ratio_mean'] = df['new_amount_month_ratio_mean'] + df['hist_amount_month_ratio_mean']\n    df['amount_month_ratio_min'] = df['new_amount_month_ratio_min'] + df['hist_amount_month_ratio_min']\n    df['amount_month_ratio_max'] = df['new_amount_month_ratio_max'] + df['hist_amount_month_ratio_max']\n    \n    df['CLV_ratio'] = df['new_CLV'] / df['hist_CLV']\n    df['CLV_sq'] = df['new_CLV'] * df['hist_CLV']\n\n    df = reduce_mem_usage(df)\n    return df\n\n\ndef modeling_xgb_cross_validation(params, X, y, nr_folds=5, verbose=0):\n    clfs = list()\n    oof_preds = np.zeros(X.shape[0])\n    # Split data with kfold\n    #kfolds = TimeSeriesSplit(n_splits=nr_folds)\n    kfolds = StratifiedKFold(n_splits=nr_folds, shuffle=True, random_state=42)\n    split_index = X[['feature_1', 'feature_2', 'feature_3']].apply(lambda x: np.log1p(x)).product(axis=1)\n    kfolds = KFold(n_splits=nr_folds, shuffle=True, random_state=42)\n    for n_fold, (trn_idx, val_idx) in enumerate(kfolds.split(X, split_index)):\n        if verbose:\n            print('no {} of {} folds'.format(n_fold, nr_folds))\n\n        X_train, y_train = X.iloc[trn_idx], y.iloc[trn_idx]\n        X_valid, y_valid = X.iloc[val_idx], y.iloc[val_idx]\n\n        model = xgb.XGBRegressor(**params)\n        model.fit(\n            X_train, y_train,\n            # eval_set=[(X_train, y_train), (X_valid, y_valid)],\n            eval_set=[(X_valid, y_valid)],\n            verbose=verbose, eval_metric='rmse',\n            early_stopping_rounds=500\n        )\n\n        clfs.append(model)\n        oof_preds[val_idx] = model.predict(X_valid, ntree_limit=model.best_ntree_limit)\n\n        del X_train, y_train, X_valid, y_valid\n        gc.collect()\n\n    score = mean_squared_error(y, oof_preds) ** .5\n    return clfs, score\n\n\ndef modeling_lgbm_cross_validation(params, X, y, nr_folds=5, verbose=0):\n    clfs = list()\n    oof_preds = np.zeros(X.shape[0])\n    # Split data with kfold\n    # kfolds = TimeSeriesSplit(n_splits=nr_folds)\n    kfolds = StratifiedKFold(n_splits=nr_folds, shuffle=True, random_state=42)\n    split_index = X[['feature_1', 'feature_2', 'feature_3']].apply(lambda x: np.log1p(x)).product(axis=1)\n    kfolds = KFold(n_splits=nr_folds, shuffle=True, random_state=42)\n    for n_fold, (trn_idx, val_idx) in enumerate(kfolds.split(X, y)):\n        if verbose:\n            print('no {} of {} folds'.format(n_fold, nr_folds))\n\n        X_train, y_train = X.iloc[trn_idx], y.iloc[trn_idx]\n        X_valid, y_valid = X.iloc[val_idx], y.iloc[val_idx]\n\n        model = lgb.LGBMRegressor(**params)\n        model.fit(\n            X_train, y_train,\n            # eval_set=[(X_train, y_train), (X_valid, y_valid)],\n            eval_set=[(X_valid, y_valid)],\n            verbose=verbose, eval_metric='rmse',\n            early_stopping_rounds=500\n        )\n\n        clfs.append(model)\n        oof_preds[val_idx] = model.predict(X_valid, num_iteration=model.best_iteration_)\n\n        del X_train, y_train, X_valid, y_valid\n        gc.collect()\n\n    score = mean_squared_error(y, oof_preds) ** .5\n    return clfs, score\n\n\ndef predict_cross_validation(test, clfs, ntree_limit=None):\n    sub_preds = np.zeros(test.shape[0])\n    for i, model in enumerate(clfs, 1):\n\n        num_tree = 10000\n        if not ntree_limit:\n            ntree_limit = num_tree\n\n        if isinstance(model, lgb.sklearn.LGBMRegressor):\n            if model.best_iteration_:\n                num_tree = min(ntree_limit, model.best_iteration_)\n\n            test_preds = model.predict(test, raw_score=True, num_iteration=num_tree)\n\n        if isinstance(model, xgb.sklearn.XGBRegressor):\n            num_tree = min(ntree_limit, model.best_ntree_limit)\n            test_preds = model.predict(test, ntree_limit=num_tree)\n\n        sub_preds += test_preds\n\n    sub_preds = sub_preds / len(clfs)\n    ret = pd.Series(sub_preds, index=test.index)\n    ret.index.name = test.index.name\n    return ret\n\n\ndef write_to_parquet(filename, df, debug=False):\n    print('write to {}: {}'.format(filename, df.shape))\n\n    # safety check\n    cols_type = df.dtypes.to_dict()\n    for col, col_type in cols_type.items():\n        if str(col_type).startswith('float16'):\n            df[col] = df[col].astype(np.float32)\n\n    df.to_parquet(filename, engine='auto', compression='snappy')\n    if debug:\n        df = pd.read_parquet(filename)\n        print('debug reload save file: {}\\n{}'.format(df.shape, df.head().T))\n\n\ndef main(debug=False):\n    num_rows = 10000 if debug else None\n    \n    with timer(\"historical transactions\"):\n        hist_df = historical_transactions(num_rows)\n        \n    with timer(\"new merchants\"):\n        new_merchant_df = new_merchant_transactions(num_rows)\n        \n    with timer(\"additional features\"):\n        df = pd.concat([new_merchant_df, hist_df], axis=1)\n        del new_merchant_df, hist_df\n        gc.collect()\n\n        train_df, test_df = train_test(num_rows)\n        train_df = train_df.join(df, how='left', on='card_id')\n        test_df = test_df.join(df, how='left', on='card_id')\n        del df\n        gc.collect()\n\n        train_df = additional_features(train_df)\n        test_df = additional_features(test_df)\n               \n    with timer(\"Run LightGBM with kfold\"):\n        excluded_features = FEATS_EXCLUDED\n        train_features = [c for c in train_df.columns if c not in excluded_features]\n        best_params = {\n            'gpu_id': 0, \n            #'n_gpus': 2, \n            'objective': 'reg:linear', \n            'eval_metric': 'rmse', \n            'silent': True, \n            'booster': 'gbtree', \n            'n_jobs': 4, \n            'n_estimators': 2500, \n            'tree_method': 'gpu_hist', \n            'grow_policy': 'lossguide', \n            'max_depth': 12, \n            'seed': 538, \n            'colsample_bylevel': 0.9, \n            'colsample_bytree': 0.8, \n            'gamma': 0.0001, \n            'learning_rate': 0.006150886706231842, \n            'max_bin': 128, \n            'max_leaves': 47, \n            'min_child_weight': 40, \n            'reg_alpha': 10.0, \n            'reg_lambda': 10.0, \n            'subsample': 0.9}\n\n        # modeling\n        nr_folds = 11\n        if debug:\n            nr_folds = 2\n        best_params.update({'n_estimators': 20000})\n        clfs = list()\n        score = 0\n        clfs, score = modeling_xgb_cross_validation(best_params,\n                                                    train_df[train_features],\n                                                    train_df['target'],\n                                                    nr_folds,\n                                                    verbose=50)\n        # save to\n        file_template = '{score:.6f}_{model_key}_cv{fold}_{timestamp}'\n        file_stem = file_template.format(\n            score=score,\n            model_key='XGB',\n            fold=nr_folds,\n            timestamp=dt.now().strftime('%Y-%m-%d-%H-%M'))\n\n        filename = 'subm_{}.csv'.format(file_stem)\n        print('save to {}'.format(filename))\n        subm = predict_cross_validation(test_df[train_features], clfs)\n        subm = subm.to_frame('target')\n        subm.to_csv(filename, index=True)\n\n\nif __name__ == \"__main__\":\n    with timer(\"Full model run\"):\n        main(debug=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"429d76240d76ed67257e111ee957a00e438ce685"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9cdd9e5263d9bc2ad04d2230591cf6889280c95"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}