{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Overview\nAs it has been pointed out in the discussion (e.g. https://www.kaggle.com/c/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/discussion/131028), most of the high scoring public kernels have a leak. That includes mine, apparently. The main cause of this is that they (intentionally or not) ignore the chronological order of the data. In this case you end up with having many cases where the future data are used by a model to predict the past, causing a leak.\n\nOne way to avoid such situations is apparently to use only the past data to predict the future, like using only ~2018 to predict 2019. Here I implemented such a validation strategy. The log_loss in the end should be around 0.5 if you are truely successful, as is always the case with the winners in the past competitions."},{"metadata":{},"cell_type":"markdown","source":"# Import Library & Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Libraries\nimport numpy as np\nimport pandas as pd\npd.set_option('max_columns', None)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('fivethirtyeight')\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder\n%matplotlib inline\nimport copy\nimport datetime\nfrom sklearn.utils import shuffle\nfrom scipy import stats\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nimport optuna\nfrom optuna.visualization import plot_optimization_history\nfrom sklearn import model_selection\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, roc_auc_score, log_loss, classification_report, confusion_matrix\nimport json\nimport ast\nimport time\nfrom sklearn import linear_model\n\n# keras\nimport keras\nfrom keras.layers import Dense\nfrom keras.models import Sequential\nfrom keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau, LambdaCallback\nfrom keras.optimizers import Adam, SGD\nfrom keras.models import Model\nfrom keras.layers import Input, Layer, Dense, Concatenate, Reshape, Dropout, merge, Add, BatchNormalization, GaussianNoise\nfrom keras.layers.embeddings import Embedding\nfrom keras import backend as K\nfrom keras.layers import Layer\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport math\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport glob\nimport gc\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import LabelEncoder\n\nprint(\"Libraries imported!\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Class\nMy modeling pipeline (still in development:D)"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class BaseModel(object):\n    \"\"\"\n    Base Model Class:\n\n    train_df : train pandas dataframe\n    test_df : test pandas dataframe\n    target : target column name (str)\n    features : list of feature names\n    categoricals : list of categorical feature names\n    n_splits : K in KFold (default is 3)\n    cv_method : options are ... KFold, StratifiedKFold, TimeSeriesSplit, GroupKFold, StratifiedGroupKFold\n    group : group feature name when GroupKFold or StratifiedGroupKFold are used\n    task : options are ... regression, multiclass, or binary\n    parameter_tuning : bool, only for LGB\n    seed : seed (int)\n    scaler : options are ... None, MinMax, Standard\n    verbose : bool\n    \"\"\"\n\n    def __init__(self, train_df, test_df, target, features, categoricals=[], \n                n_splits=3, cv_method=\"KFold\", group=None, task=\"regression\", \n                parameter_tuning=False, seed=42, scaler=None, verbose=True):\n        self.train_df = train_df\n        self.test_df = test_df\n        self.target = target\n        self.features = features\n        self.n_splits = n_splits\n        self.categoricals = categoricals\n        self.cv_method = cv_method\n        self.group = group\n        self.task = task\n        self.parameter_tuning = parameter_tuning\n        self.seed = seed\n        self.scaler = scaler\n        self.cv = self.get_cv()\n        self.verbose = verbose\n        self.params = self.get_params()\n        self.y_pred, self.score, self.model, self.oof, self.y_val, self.fi_df = self.fit()\n\n    def train_model(self, train_set, val_set):\n        raise NotImplementedError\n\n    def get_params(self):\n        raise NotImplementedError\n\n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        raise NotImplementedError\n\n    def convert_x(self, x):\n        return x\n\n    def calc_metric(self, y_true, y_pred): # this may need to be changed based on the metric of interest\n        if self.task == \"multiclass\":\n            return log_loss(y_true, y_pred)\n        elif self.task == \"binary\":\n            return log_loss(y_true, y_pred)\n#             return roc_auc_score(y_true, y_pred)\n        elif self.task == \"regression\":\n            return np.sqrt(mean_squared_error(y_true, y_pred))\n\n    def get_cv(self):\n        if self.cv_method == \"KFold\":\n            cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)\n            return cv.split(self.train_df)\n        elif self.cv_method == \"StratifiedKFold\":\n            cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)\n            return cv.split(self.train_df, self.train_df[self.target])\n        elif self.cv_method == \"TimeSeriesSplit\":\n            cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)\n            return cv.split(self.train_df)\n\n    def fit(self):\n        # initialize\n        oof_pred = np.zeros((self.train_df.shape[0], ))\n        y_vals = np.zeros((self.train_df.shape[0], ))\n        y_pred = np.zeros((self.test_df.shape[0], ))\n        if self.group is not None:\n            if self.group in self.features:\n                self.features.remove(self.group)\n            if self.group in self.categoricals:\n                self.categoricals.remove(self.group)\n        fi = np.zeros((self.n_splits, len(self.features)))\n\n        # scaling, if necessary\n        if self.scaler is not None:\n            # fill NaN\n            numerical_features = [f for f in self.features if f not in self.categoricals]\n            self.train_df[numerical_features] = self.train_df[numerical_features].fillna(self.train_df[numerical_features].median())\n            self.test_df[numerical_features] = self.test_df[numerical_features].fillna(self.test_df[numerical_features].median())\n            self.train_df[self.categoricals] = self.train_df[self.categoricals].fillna(self.train_df[self.categoricals].mode().iloc[0])\n            self.test_df[self.categoricals] = self.test_df[self.categoricals].fillna(self.test_df[self.categoricals].mode().iloc[0])\n\n            # scaling\n            if self.scaler == \"MinMax\":\n                scaler = MinMaxScaler()\n            elif self.scaler == \"Standard\":\n                scaler = StandardScaler()\n            df = pd.concat([self.train_df[numerical_features], self.test_df[numerical_features]], ignore_index=True)\n            scaler.fit(df[numerical_features])\n            x_test = self.test_df.copy()\n            x_test[numerical_features] = scaler.transform(x_test[numerical_features])\n            x_test = [np.absolute(x_test[i]) for i in self.categoricals] + [x_test[numerical_features]]\n        else:\n            x_test = self.test_df[self.features]\n\n        # fitting with out of fold\n        for fold, (train_idx, val_idx) in enumerate(self.cv):\n            # train test split\n            x_train, x_val = self.train_df.loc[train_idx, self.features], self.train_df.loc[val_idx, self.features]\n            y_train, y_val = self.train_df.loc[train_idx, self.target], self.train_df.loc[val_idx, self.target]\n\n            # fitting & get feature importance\n            if self.scaler is not None:\n                x_train[numerical_features] = scaler.transform(x_train[numerical_features])\n                x_val[numerical_features] = scaler.transform(x_val[numerical_features])\n                x_train = [np.absolute(x_train[i]) for i in self.categoricals] + [x_train[numerical_features]]\n                x_val = [np.absolute(x_val[i]) for i in self.categoricals] + [x_val[numerical_features]]\n            train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)\n            model, importance = self.train_model(train_set, val_set)\n            fi[fold, :] = importance\n            conv_x_val = self.convert_x(x_val)\n            y_vals[val_idx] = y_val\n            oof_pred[val_idx] = model.predict(conv_x_val).reshape(oof_pred[val_idx].shape)\n            x_test = self.convert_x(x_test)\n            y_pred += model.predict(x_test).reshape(y_pred.shape) / self.n_splits\n            print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))\n        \n        # feature importance data frame\n        fi_df = pd.DataFrame()\n        for n in np.arange(self.n_splits):\n            tmp = pd.DataFrame()\n            tmp[\"features\"] = self.features\n            tmp[\"importance\"] = fi[n, :]\n            tmp[\"fold\"] = n\n            fi_df = pd.concat([fi_df, tmp], ignore_index=True)\n        gfi = fi_df[[\"features\", \"importance\"]].groupby([\"features\"]).mean().reset_index()\n        fi_df = fi_df.merge(gfi, on=\"features\", how=\"left\", suffixes=('', '_mean'))\n\n        # outputs\n        loss_score = self.calc_metric(y_vals, oof_pred)\n        if self.verbose:\n            print('Our oof loss score is: ', loss_score)\n        return y_pred, loss_score, model, oof_pred, y_vals, fi_df\n\n    def plot_feature_importance(self, rank_range=[1, 50]):\n        # plot\n        fig, ax = plt.subplots(1, 1, figsize=(10, 20))\n        sorted_df = self.fi_df.sort_values(by = \"importance_mean\", ascending=False).reset_index().iloc[self.n_splits * (rank_range[0]-1) : self.n_splits * rank_range[1]]\n        sns.barplot(data=sorted_df, x =\"importance\", y =\"features\", orient='h')\n        ax.set_xlabel(\"feature importance\")\n        ax.spines['top'].set_visible(False)\n        ax.spines['right'].set_visible(False)\n        return sorted_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class LgbModel(BaseModel):\n    \"\"\"\n    LGB wrapper\n\n    \"\"\"\n\n    def train_model(self, train_set, val_set):\n        verbosity = 100 if self.verbose else 0\n        model = lgb.train(self.params, train_set, num_boost_round = 3000, valid_sets=[train_set, val_set], verbose_eval=verbosity)\n        fi = model.feature_importance(importance_type=\"gain\")\n        return model, fi\n\n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)\n        val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)\n        return train_set, val_set\n\n    def get_params(self):\n        # fast fit parameters\n        params = {\n          'num_leaves': 127,\n          'objective': self.task,\n          'min_data_in_leaf': 50,\n          'max_depth': -1,\n          'learning_rate': 0.005,\n          \"boosting_type\": \"gbdt\",\n          \"bagging_seed\": 11,\n          \"verbosity\": -1,\n          'random_state': 42,\n         }\n\n        # list is here: https://lightgbm.readthedocs.io/en/latest/Parameters.html\n        if self.task == \"regression\":\n            params[\"metric\"] = \"rmse\"\n        elif self.task == \"binary\":\n            params[\"metric\"] = \"binary_logloss\" # auc\n        elif self.task == \"multiclass\":\n            params[\"metric\"] = \"multi_logloss\" # cross_entropy, auc_mu\n        \n        # Bayesian Optimization by Optuna\n        if self.parameter_tuning == True:\n            # define objective function\n            def objective(trial):\n                # train, test split\n                train_x, test_x, train_y, test_y = train_test_split(self.train_df[self.features], \n                                                                    self.train_df[self.target],\n                                                                    test_size=0.3, random_state=self.seed)\n                dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)\n                dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)\n\n                # parameters to be explored\n                hyperparams = {'num_leaves': trial.suggest_int('num_leaves', 24, 1024),\n                        'boosting_type': 'gbdt',\n                        'objective': params[\"objective\"],\n                        'metric': params[\"metric\"],\n                        'max_depth': trial.suggest_int('max_depth', 4, 16),\n                        'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),\n                        'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),\n                        'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),\n                        'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),\n                        'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),\n                        'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),\n                        'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),\n                        'early_stopping_rounds': 100\n                        }\n\n                # LGB\n                model = lgb.train(hyperparams, dtrain, valid_sets=dtest, verbose_eval=500)\n                pred = model.predict(test_x)\n                if (self.task == \"binary\") | (self.task == \"multiclass\"):\n                    return log_loss(test_y, pred)\n                elif self.task == \"regression\":\n                    return np.sqrt(mean_squared_error(test_y, pred))\n\n            # run optimization\n            study = optuna.create_study(direction='minimize')\n            study.optimize(objective, n_trials=50)\n\n            print('Number of finished trials: {}'.format(len(study.trials)))\n            print('Best trial:')\n            trial = study.best_trial\n            print('  Value: {}'.format(trial.value))\n            print('  Params: ')\n            for key, value in trial.params.items():\n                print('    {}: {}'.format(key, value))\n\n            params = trial.params\n\n            # lower learning rate for better accuracy\n            params[\"learning_rate\"] = 0.001\n\n            # plot history\n            plot_optimization_history(study)\n\n        return params","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dict = {}\nfor i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/*'):\n    name = i.split('/')[-1].split('.')[0]\n    if name != 'WTeamSpellings':\n        data_dict[name] = pd.read_csv(i)\n    else:\n        data_dict[name] = pd.read_csv(i, encoding='cp1252')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Overview"},{"metadata":{},"cell_type":"markdown","source":"NCAAW20 has less data than NCAAM20. It may be easier for us to start with NCAAW20."},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dict.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'Cities'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeamSpellings'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WSeasons'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeams'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneyCompactResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WGameCities'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'Conferences'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneySeeds'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get int from seed\ndata_dict['WNCAATourneySeeds']['Seed'] = data_dict['WNCAATourneySeeds']['Seed'].apply(lambda x: int(x[1:3]))\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneySlots'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeamConferences'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneyDetailedResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WRegularSeasonDetailedResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WRegularSeasonCompactResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# let's also have a look at test\ntest = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# format ID\ntest = test.drop(['Pred'], axis=1)\ntest['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))\ntest['WTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[1]))\ntest['LTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[2]))\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data processing and feature engineering.\n\nThe main idea is to extract features, which could be useful to understand how much one team is better than another one."},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge tables ============\ntrain = data_dict['WNCAATourneyCompactResults'] # use compact data only for now\n\n# # compact <- detailed (Tourney files)\n# train = pd.merge(data_dict['MNCAATourneyCompactResults'], data_dict['MNCAATourneyDetailedResults'], how='left',\n#              on=['Season', 'DayNum', 'WTeamID', 'WScore', 'LTeamID', 'LScore', 'WLoc', 'NumOT'])\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train =================================\n# merge with Game Cities\ngameCities = pd.merge(data_dict['WGameCities'], data_dict['Cities'], how='left', on=['CityID'])\ncols_to_use = gameCities.columns.difference(train.columns).tolist() + [\"Season\", \"WTeamID\", \"LTeamID\"]\ntrain = train.merge(gameCities[cols_to_use], how=\"left\", on=[\"Season\", \"WTeamID\", \"LTeamID\"])\ntrain.head()\n\n# merge with WSeasons\ncols_to_use = data_dict[\"WSeasons\"].columns.difference(train.columns).tolist() + [\"Season\"]\ntrain = train.merge(data_dict[\"WSeasons\"][cols_to_use], how=\"left\", on=[\"Season\"])\ntrain.head()\n\n# merge with WTeams\ncols_to_use = data_dict[\"WTeams\"].columns.difference(train.columns).tolist()\ntrain = train.merge(data_dict[\"WTeams\"][cols_to_use], how=\"left\", left_on=[\"WTeamID\"], right_on=[\"TeamID\"])\ntrain.drop(['TeamID'], axis=1, inplace=True)\ntrain = train.merge(data_dict[\"WTeams\"][cols_to_use], how=\"left\", left_on=[\"LTeamID\"], right_on=[\"TeamID\"], suffixes=('_W', '_L'))\ntrain.drop(['TeamID'], axis=1, inplace=True)\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with WNCAATourneySeeds\ncols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(train.columns).tolist() + ['Season']\ntrain = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                    how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])\ntrain.drop(['TeamID'], axis=1, inplace=True)\ntrain = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                    how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))\ntrain.drop(['TeamID'], axis=1, inplace=True)\n\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test =================================\n# merge with Game Cities\ncols_to_use = gameCities.columns.difference(test.columns).tolist() + [\"Season\", \"WTeamID\", \"LTeamID\"]\ntest = test.merge(gameCities[cols_to_use].drop_duplicates(subset=[\"Season\", \"WTeamID\", \"LTeamID\"]),\n                  how=\"left\", on=[\"Season\", \"WTeamID\", \"LTeamID\"])\ndel gameCities\ngc.collect()\ntest.head()\n\n# merge with WSeasons\ncols_to_use = data_dict[\"WSeasons\"].columns.difference(test.columns).tolist() + [\"Season\"]\ntest = test.merge(data_dict[\"WSeasons\"][cols_to_use].drop_duplicates(subset=[\"Season\"]),\n                  how=\"left\", on=[\"Season\"])\ntest.head()\n\n# merge with WTeams\ncols_to_use = data_dict[\"WTeams\"].columns.difference(test.columns).tolist()\ntest = test.merge(data_dict[\"WTeams\"][cols_to_use].drop_duplicates(subset=[\"TeamID\"]),\n                  how=\"left\", left_on=[\"WTeamID\"], right_on=[\"TeamID\"])\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest = test.merge(data_dict[\"WTeams\"][cols_to_use].drop_duplicates(subset=[\"TeamID\"]),\n                  how=\"left\", left_on=[\"LTeamID\"], right_on=[\"TeamID\"], suffixes=('_W', '_L'))\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest.head()\n\n# merge with WNCAATourneySeeds\ncols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(test.columns).tolist() + ['Season']\ntest = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                  how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                  how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))\ntest.drop(['TeamID'], axis=1, inplace=True)\n\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist()]\nprint(not_exist_in_test)\ntrain = train.drop(not_exist_in_test, axis=1)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sort by time (just to make sure the ordering...)\nTo ensure no leak, we convert \"DayZero\" column to datetime object such that we explicitly have an order in our data."},{"metadata":{"trusted":true},"cell_type":"code","source":"# to datetime format\ntrain[\"DayZero\"] = pd.to_datetime(train[\"DayZero\"], infer_datetime_format=True)\n\n# sort by date\ntrain = train.sort_values(by=[\"DayZero\", \"Season\", \"DayNum\"]).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CAUTION: dealing with 'WRegularSeasonCompactResults'\nThis part is very likely to be the cause of the leak. You cannot merge aggregated data containing future results with your past data."},{"metadata":{"trusted":true},"cell_type":"code","source":"# compact <- detailed (regular season files)\nregularSeason = data_dict['WRegularSeasonCompactResults']\n# regularSeason = pd.merge(data_dict['WRegularSeasonCompactResults'], data_dict['WRegularSeasonDetailedResults'], how='left',\n#              on=['Season', 'DayNum', 'WTeamID', 'WScore', 'LTeamID', 'LScore', 'WLoc', 'NumOT'])\nprint(regularSeason.shape)\nregularSeason.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with regularSeason using only the previous data\ndef merge_regularSeason(df, regularSeason):\n    df_new = pd.DataFrame()\n    for i, season in enumerate(df[\"Season\"].unique()):\n        print(season)\n        if season <= 1998:\n            continue\n            \n        # split winners and losers (make sure not to use the future data!)\n        team_win_score = regularSeason.loc[regularSeason[\"Season\"] <= season, :].groupby(['WTeamID']).agg({'WScore':['sum', 'count', 'var']}).reset_index()\n        team_win_score.columns = [' '.join(col).strip() for col in team_win_score.columns.values]\n        team_loss_score = regularSeason.loc[regularSeason[\"Season\"] <= season, :].groupby(['LTeamID']).agg({'LScore':['sum', 'count', 'var']}).reset_index()\n        team_loss_score.columns = [' '.join(col).strip() for col in team_loss_score.columns.values]\n        \n        # merge with train \n        team_win_score[\"Season\"] = season\n        team_loss_score[\"Season\"] = season\n        df_fold = df.loc[df[\"Season\"] == season, :].reset_index(drop=True)\n        df_fold = df_fold.merge(team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])\n        df_fold = df_fold.merge(team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])\n        df_fold = df_fold.merge(team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])\n        df_fold = df_fold.merge(team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])\n        df_fold.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\n        \n        # restore\n        df_new = pd.concat([df_new, df_fold], ignore_index=True)\n    return df_new\n\ntrain = merge_regularSeason(train, regularSeason)\ntest = merge_regularSeason(test, regularSeason)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preprocess\ndef preprocess(df):\n    df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']\n    df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']\n    df['x_count'] = df['WScore count_x'] + df['LScore count_y']\n    df['y_count'] = df['WScore count_y'] + df['WScore count_x']\n    df['x_var'] = df['WScore var_x'] + df['LScore var_x']\n    df['y_var'] = df['WScore var_y'] + df['LScore var_y']\n    return df\ntrain = preprocess(train)\ntest = preprocess(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make winner and loser train\ntrain_win = train.copy()\ntrain_los = train.copy()\ntrain_win = train_win[['DayNum', 'DayZero', 'Season', 'Seed_W', 'Seed_L', 'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntrain_los = train_los[['DayNum', 'DayZero', 'Season', 'Seed_L', 'Seed_W', 'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntrain_win.columns = ['DayNum', 'DayZero', 'Season', 'Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']\ntrain_los.columns = ['DayNum', 'DayZero', 'Season', 'Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']\n\n# same processing for test\ntest = test[['ID', 'Season', 'Seed_W', 'Seed_L', 'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntest.columns = ['ID', 'Season', 'Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature enginnering\ndef feature_engineering(df):\n    df['Seed_diff'] = df['Seed_1'] - df['Seed_2']\n    df['Seed_ratio'] = df['Seed_1'] / df['Seed_2']\n    df['Score_ratio'] = df['Score_1'] / df['Score_2']\n    df['Count_ratio'] = df['Count_1'] / df['Count_2']\n    df['Var_ratio'] = df['Var_1'] / df['Var_2']\n    df['Mean_score1'] = df['Score_1'] / df['Count_1']\n    df['Mean_score2'] = df['Score_2'] / df['Count_2']\n    df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']\n    df['Mean_score_ratio'] = df['Mean_score1'] / df['Mean_score2']\n    df = df.drop(['Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2'], axis=1)\n    return df\ntrain_win = feature_engineering(train_win)\ntrain_los = feature_engineering(train_los)\ntest = feature_engineering(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_win[\"result\"] = 1\nprint(train_win.shape)\ntrain_win.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_los[\"result\"] = 0\nprint(train_los.shape)\ntrain_los.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.concat([train_win, train_los], ignore_index=True)\ndata = data.sort_values(by=['DayZero', 'Season', 'DayNum']).reset_index(drop=True)\nprint(data.shape)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict & Make Submission File"},{"metadata":{},"cell_type":"markdown","source":"## Model training\nWe use TimeSeriesSplit to ensure no leakage due to flipping time, like training using ~2018 to predict 2019 and so on.[](http://)"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'result'\nfeatures = data.columns.values.tolist()\ndropcols = [target,'DayNum', 'Season', 'DayZero']\nfeatures = [f for f in features if f not in dropcols]\nprint(features)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fit LGB"},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"Season\"].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict 2015 by using ~2014, for example\nmodels = {}\ntest[\"Pred\"] = 0.5\npred = np.zeros(test.shape[0])\nfor season in test[\"Season\"].unique():\n    print(f\"Predicting {season}...\")\n    lgbm = LgbModel(data.loc[data[\"Season\"] < season, :], test.loc[test[\"Season\"] == season, :], target, features, categoricals=[], n_splits=10, \n                    cv_method=\"TimeSeriesSplit\", group=None, task=\"binary\", scaler=None, verbose=True)\n    models[season] = lgbm\n    test.loc[test[\"Season\"] == season, \"Pred\"] = lgbm.y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature importance\nfi_df = models[2019].plot_feature_importance()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = test['Pred']\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}