{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Overview\n#### This kernel is based on [2020 Starter Kernel Women](https://www.kaggle.com/hiromoon166/2020-women-s-starter-kernel)\n#### I added my GBDT (+NN) pipelines to see the feature importance from LGB and CatBoost.\n\nAlso see my starter for mens' games: https://www.kaggle.com/code1110/ncaam20-eda-and-lgb-catb-starter"},{"metadata":{},"cell_type":"markdown","source":"# Import Library & Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Libraries\nimport numpy as np\nimport pandas as pd\npd.set_option('max_columns', None)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('fivethirtyeight')\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder\n%matplotlib inline\nimport copy\nimport datetime\nfrom sklearn.utils import shuffle\nfrom scipy import stats\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold, cross_val_score, GridSearchCV, RepeatedStratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nimport optuna\nfrom optuna.visualization import plot_optimization_history\nfrom sklearn import model_selection\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, roc_auc_score, log_loss, classification_report, confusion_matrix\nimport json\nimport ast\nimport time\nfrom sklearn import linear_model\n\n# keras\nimport keras\nfrom keras.layers import Dense\nfrom keras.models import Sequential\nfrom keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau, LambdaCallback\nfrom keras.optimizers import Adam, SGD\nfrom keras.models import Model\nfrom keras.layers import Input, Layer, Dense, Concatenate, Reshape, Dropout, merge, Add, BatchNormalization, GaussianNoise\nfrom keras.layers.embeddings import Embedding\nfrom keras import backend as K\nfrom keras.layers import Layer\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport math\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport glob\nimport gc\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import LabelEncoder\n\nprint(\"Libraries imported!\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseModel(object):\n    \"\"\"\n    Base Model Class\n\n    \"\"\"\n\n    def __init__(self, train_df, test_df, target, features, categoricals=[], \n                n_splits=3, cv_method=\"KFold\", group=None, task=\"regression\", \n                parameter_tuning=False, scaler=None, verbose=True):\n        self.train_df = train_df\n        self.test_df = test_df\n        self.target = target\n        self.features = features\n        self.n_splits = n_splits\n        self.categoricals = categoricals\n        self.cv_method = cv_method\n        self.group = group\n        self.task = task\n        self.parameter_tuning = parameter_tuning\n        self.scaler = scaler\n        self.cv = self.get_cv()\n        self.verbose = verbose\n        self.params = self.get_params()\n        self.y_pred, self.score, self.model, self.oof, self.y_val, self.fi_df = self.fit()\n\n    def train_model(self, train_set, val_set):\n        raise NotImplementedError\n\n    def get_params(self):\n        raise NotImplementedError\n\n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        raise NotImplementedError\n\n    def convert_x(self, x):\n        return x\n\n    def calc_metric(self, y_true, y_pred): # this may need to be changed based on the metric of interest\n        if self.task == \"classification\":\n            return log_loss(y_true, y_pred)\n        elif self.task == \"regression\":\n            return np.sqrt(mean_squared_error(y_true, y_pred))\n\n    def get_cv(self):\n        if self.cv_method == \"KFold\":\n            cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n            return cv.split(self.train_df)\n        elif self.cv_method == \"StratifiedKFold\":\n            cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n            return cv.split(self.train_df, self.train_df[self.target])\n        elif self.cv_method == \"TimeSeriesSplit\":\n            cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)\n            return cv.split(self.train_df)\n        elif self.cv_method == \"GroupKFold\":\n            cv = GroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n            return cv.split(self.train_df, self.train_df[self.target], self.group)\n        elif self.cv_method == \"StratifiedGroupKFold\":\n            cv = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n            return cv.split(self.train_df, self.train_df[self.target], self.group)\n\n    def fit(self):\n        # initialize\n        oof_pred = np.zeros((self.train_df.shape[0], ))\n        y_vals = np.zeros((self.train_df.shape[0], ))\n        y_pred = np.zeros((self.test_df.shape[0], ))\n        if self.group is not None:\n            if self.group in self.features:\n                self.features.remove(self.group)\n            if self.group in self.categoricals:\n                self.categoricals.remove(self.group)\n        fi = np.zeros((self.n_splits, len(self.features)))\n\n        # scaling, if necessary\n        if self.scaler is not None:\n            # fill NaN\n            numerical_features = [f for f in self.features if f not in self.categoricals]\n            self.train_df[numerical_features] = self.train_df[numerical_features].fillna(self.train_df[numerical_features].median())\n            self.test_df[numerical_features] = self.test_df[numerical_features].fillna(self.test_df[numerical_features].median())\n            self.train_df[self.categoricals] = self.train_df[self.categoricals].fillna(self.train_df[self.categoricals].mode().iloc[0])\n            self.test_df[self.categoricals] = self.test_df[self.categoricals].fillna(self.test_df[self.categoricals].mode().iloc[0])\n\n            # scaling\n            if self.scaler == \"MinMax\":\n                scaler = MinMaxScaler()\n            elif self.scaler == \"Standard\":\n                scaler = StandardScaler()\n            df = pd.concat([self.train_df[numerical_features], self.test_df[numerical_features]], ignore_index=True)\n            scaler.fit(df[numerical_features])\n            x_test = self.test_df.copy()\n            x_test[numerical_features] = scaler.transform(x_test[numerical_features])\n            x_test = [np.absolute(x_test[i]) for i in self.categoricals] + [x_test[numerical_features]]\n        else:\n            x_test = self.test_df[self.features]\n            \n        # fitting with out of fold\n        for fold, (train_idx, val_idx) in enumerate(self.cv):\n            # train test split\n            x_train, x_val = self.train_df.loc[train_idx, self.features], self.train_df.loc[val_idx, self.features]\n            y_train, y_val = self.train_df.loc[train_idx, self.target], self.train_df.loc[val_idx, self.target]\n\n            # fitting & get feature importance\n            if self.scaler is not None:\n                x_train[numerical_features] = scaler.transform(x_train[numerical_features])\n                x_val[numerical_features] = scaler.transform(x_val[numerical_features])\n                x_train = [np.absolute(x_train[i]) for i in self.categoricals] + [x_train[numerical_features]]\n                x_val = [np.absolute(x_val[i]) for i in self.categoricals] + [x_val[numerical_features]]\n            train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)\n            model, importance = self.train_model(train_set, val_set)\n            fi[fold, :] = importance\n            conv_x_val = self.convert_x(x_val)\n            y_vals[val_idx] = y_val\n            oof_pred[val_idx] = model.predict(conv_x_val).reshape(oof_pred[val_idx].shape)\n            x_test = self.convert_x(x_test)\n            y_pred += model.predict(x_test).reshape(y_pred.shape) / self.n_splits\n            print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))\n\n        # feature importance data frame\n        fi_df = pd.DataFrame()\n        for n in np.arange(self.n_splits):\n            tmp = pd.DataFrame()\n            tmp[\"features\"] = self.features\n            tmp[\"importance\"] = fi[n, :]\n            tmp[\"fold\"] = n\n            fi_df = pd.concat([fi_df, tmp], ignore_index=True)\n        gfi = fi_df[[\"features\", \"importance\"]].groupby([\"features\"]).mean().reset_index()\n        fi_df = fi_df.merge(gfi, on=\"features\", how=\"left\", suffixes=('', '_mean'))\n\n        # outputs\n        loss_score = self.calc_metric(self.train_df[self.target], oof_pred)\n        if self.verbose:\n            print('Our oof loss score is: ', loss_score)\n        return y_pred, loss_score, model, oof_pred, y_vals, fi_df\n\n    def plot_feature_importance(self, rank_range=[1, 50]):\n        # plot\n        fig, ax = plt.subplots(1, 1, figsize=(10, 20))\n        sorted_df = self.fi_df.sort_values(by = \"importance_mean\", ascending=False).reset_index().iloc[self.n_splits * (rank_range[0]-1) : self.n_splits * rank_range[1]]\n        sns.barplot(data=sorted_df, x =\"importance\", y =\"features\", orient='h')\n        ax.set_xlabel(\"feature importance\")\n        ax.spines['top'].set_visible(False)\n        ax.spines['right'].set_visible(False)\n        return sorted_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LgbModel(BaseModel):\n    \"\"\"\n    LGB wrapper\n\n    \"\"\"\n\n    def train_model(self, train_set, val_set):\n        verbosity = 100 if self.verbose else 0\n        model = lgb.train(self.params, train_set, num_boost_round = 5000, valid_sets=[train_set, val_set], verbose_eval=verbosity)\n        fi = model.feature_importance(importance_type=\"gain\")\n        return model, fi\n\n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)\n        val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)\n        return train_set, val_set\n\n    def get_params(self):\n        # params from https://www.kaggle.com/vbmokin/mm-2020-ncaam-simple-lightgbm-on-kfold-tuning\n        params = {\n          'num_leaves': 127,\n          'min_data_in_leaf': 50,\n          'max_depth': -1,\n          'learning_rate': 0.005,\n          \"boosting_type\": \"gbdt\",\n          \"bagging_seed\": 11,\n          \"verbosity\": -1,\n          'random_state': 42,\n         }\n        \n        if self.task == \"regression\":\n            params[\"objective\"] = \"regression\"\n            params[\"metric\"] = \"rmse\"\n        elif self.task == \"classification\":\n            params[\"objective\"] = \"binary\"\n            params[\"metric\"] = \"binary_logloss\"\n        \n        # Bayesian Optimization by Optuna\n        if self.parameter_tuning == True:\n            # define objective function\n            def objective(trial):\n                # train, test split\n                train_x, test_x, train_y, test_y = train_test_split(self.train_df[self.features], \n                                                                    self.train_df[self.target],\n                                                                    test_size=0.3, random_state=42)\n                dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)\n                dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)\n\n                # parameters to be explored\n                hyperparams = {'num_leaves': trial.suggest_int('num_leaves', 24, 1024),\n                        'boosting_type': 'gbdt',\n                        'objective': params[\"objective\"],\n                        'metric': params[\"metric\"],\n                        'max_depth': trial.suggest_int('max_depth', 4, 16),\n                        'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),\n                        'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),\n                        'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),\n                        'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),\n                        'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),\n                        'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),\n                        'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),\n                        'early_stopping_rounds': 100\n                        }\n\n                # LGB\n                model = lgb.train(hyperparams, dtrain, valid_sets=dtest, verbose_eval=500)\n                pred = model.predict(test_x)\n                if self.task == \"classification\":\n                    return log_loss(test_y, pred)\n                elif self.task == \"regression\":\n                    return np.sqrt(mean_squared_error(test_y, pred))\n\n            # run optimization\n            study = optuna.create_study(direction='minimize')\n            study.optimize(objective, n_trials=50)\n\n            print('Number of finished trials: {}'.format(len(study.trials)))\n            print('Best trial:')\n            trial = study.best_trial\n            print('  Value: {}'.format(trial.value))\n            print('  Params: ')\n            for key, value in trial.params.items():\n                print('    {}: {}'.format(key, value))\n\n            params = trial.params\n\n            # lower learning rate for better accuracy\n            params[\"learning_rate\"] = 0.001\n\n            # plot history\n            plot_optimization_history(study)\n\n        return params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CatbModel(BaseModel):\n    \"\"\"\n    CatBoost wrapper\n    \"\"\"\n    def train_model(self, train_set, val_set):\n        verbosity = 100 if self.verbose else 0\n        if self.task == \"regression\":\n            model = CatBoostRegressor(**self.params)\n        elif self.task == \"classification\":\n            model = CatBoostClassifier(**self.params)\n        model.fit(train_set['X'], train_set['y'], eval_set=(val_set['X'], val_set['y']),\n            verbose=verbosity, cat_features=self.categoricals)\n        return model, model.get_feature_importance()\n\n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        train_set = {'X': x_train, 'y': y_train}\n        val_set = {'X': x_val, 'y': y_val}\n        return train_set, val_set\n\n    def get_params(self):\n        params = { 'task_type': \"CPU\",\n                   'learning_rate': 0.01,\n                   'iterations': 1000,\n                   'random_seed': 42,\n                   'use_best_model': True\n                    }\n        if self.task == \"regression\":\n            params[\"loss_function\"] = \"RMSE\"\n        elif self.task == \"classification\":\n            params[\"loss_function\"] = \"Logloss\"\n        return params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Mish activation\nclass Mish(Layer):\n    def __init__(self, **kwargs):\n        super(Mish, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        super(Mish, self).build(input_shape)\n\n    def call(self, x):\n        return x * K.tanh(K.softplus(x))\n\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\nfrom keras import backend as K\n\n# LayerNormalization\nclass LayerNormalization(keras.layers.Layer):\n\n    def __init__(self,\n                 center=True,\n                 scale=True,\n                 epsilon=None,\n                 gamma_initializer='ones',\n                 beta_initializer='zeros',\n                 gamma_regularizer=None,\n                 beta_regularizer=None,\n                 gamma_constraint=None,\n                 beta_constraint=None,\n                 **kwargs):\n        \"\"\"Layer normalization layer\n        See: [Layer Normalization](https://arxiv.org/pdf/1607.06450.pdf)\n        :param center: Add an offset parameter if it is True.\n        :param scale: Add a scale parameter if it is True.\n        :param epsilon: Epsilon for calculating variance.\n        :param gamma_initializer: Initializer for the gamma weight.\n        :param beta_initializer: Initializer for the beta weight.\n        :param gamma_regularizer: Optional regularizer for the gamma weight.\n        :param beta_regularizer: Optional regularizer for the beta weight.\n        :param gamma_constraint: Optional constraint for the gamma weight.\n        :param beta_constraint: Optional constraint for the beta weight.\n        :param kwargs:\n        \"\"\"\n        super(LayerNormalization, self).__init__(**kwargs)\n        self.supports_masking = True\n        self.center = center\n        self.scale = scale\n        if epsilon is None:\n            epsilon = K.epsilon() * K.epsilon()\n        self.epsilon = epsilon\n        self.gamma_initializer = keras.initializers.get(gamma_initializer)\n        self.beta_initializer = keras.initializers.get(beta_initializer)\n        self.gamma_regularizer = keras.regularizers.get(gamma_regularizer)\n        self.beta_regularizer = keras.regularizers.get(beta_regularizer)\n        self.gamma_constraint = keras.constraints.get(gamma_constraint)\n        self.beta_constraint = keras.constraints.get(beta_constraint)\n        self.gamma, self.beta = None, None\n\n    def get_config(self):\n        config = {\n            'center': self.center,\n            'scale': self.scale,\n            'epsilon': self.epsilon,\n            'gamma_initializer': keras.initializers.serialize(self.gamma_initializer),\n            'beta_initializer': keras.initializers.serialize(self.beta_initializer),\n            'gamma_regularizer': keras.regularizers.serialize(self.gamma_regularizer),\n            'beta_regularizer': keras.regularizers.serialize(self.beta_regularizer),\n            'gamma_constraint': keras.constraints.serialize(self.gamma_constraint),\n            'beta_constraint': keras.constraints.serialize(self.beta_constraint),\n        }\n        base_config = super(LayerNormalization, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\n    def compute_mask(self, inputs, input_mask=None):\n        return input_mask\n\n    def build(self, input_shape):\n        shape = input_shape[-1:]\n        if self.scale:\n            self.gamma = self.add_weight(\n                shape=shape,\n                initializer=self.gamma_initializer,\n                regularizer=self.gamma_regularizer,\n                constraint=self.gamma_constraint,\n                name='gamma',\n            )\n        if self.center:\n            self.beta = self.add_weight(\n                shape=shape,\n                initializer=self.beta_initializer,\n                regularizer=self.beta_regularizer,\n                constraint=self.beta_constraint,\n                name='beta',\n            )\n        super(LayerNormalization, self).build(input_shape)\n\n    def call(self, inputs, training=None):\n        mean = K.mean(inputs, axis=-1, keepdims=True)\n        variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True)\n        std = K.sqrt(variance + self.epsilon)\n        outputs = (inputs - mean) / std\n        if self.scale:\n            outputs *= self.gamma\n        if self.center:\n            outputs += self.beta\n        return outputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class NeuralNetworkModel(BaseModel):\n    \"\"\"\n    MLP wrapper: for now not so flexible\n\n    \"\"\"\n\n    def train_model(self, train_set, val_set):\n        # MLP model\n        inputs = []\n        embeddings = []\n        embedding_out_dim = self.params['embedding_out_dim']\n        n_neuron = self.params['hidden_units']\n        for i in self.categoricals:\n            input_ = Input(shape=(1,))\n            embedding = Embedding(int(np.absolute(self.train_df[i]).max() + 1), embedding_out_dim, input_length=1)(input_)\n            embedding = Reshape(target_shape=(embedding_out_dim,))(embedding)\n            inputs.append(input_)\n            embeddings.append(embedding)\n        input_numeric = Input(shape=(len(self.features) - len(self.categoricals),))\n        embedding_numeric = Dense(n_neuron)(input_numeric)\n        embedding_numeric = Mish()(embedding_numeric)\n        inputs.append(input_numeric)\n        embeddings.append(embedding_numeric)\n        x = Concatenate()(embeddings)\n        for i in np.arange(self.params['hidden_layers'] - 1):\n            x = Dense(n_neuron // (2 * (i+1)))(x)\n            x = Mish()(x)\n            x = Dropout(self.params['hidden_dropout'])(x)\n            x = LayerNormalization()(x)\n        if self.task == \"regression\":\n            out = Dense(1, activation=\"linear\", name = \"out\")(x)\n            loss = \"mse\"\n        elif self.task == \"classification\":\n            out = Dense(1, activation='sigmoid', name = 'out')(x)\n            loss = \"binary_crossentropy\"\n        model = Model(inputs=inputs, outputs=out)\n\n        # compile\n        model.compile(loss=loss, optimizer=Adam(lr=1e-04, beta_1=0.9, beta_2=0.999, decay=1e-04))\n\n        # callbacks\n        er = EarlyStopping(patience=10, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')\n        ReduceLR = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=7, verbose=1, epsilon=1e-4, mode='min')\n        model.fit(train_set['X'], train_set['y'], callbacks=[er, ReduceLR],\n                            epochs=self.params['epochs'], batch_size=self.params['batch_size'],\n                            validation_data=[val_set['X'], val_set['y']])\n        fi = np.zeros(len(self.features)) # no feature importance computed\n        return model, fi\n\n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        train_set = {'X': x_train, 'y': y_train}\n        val_set = {'X': x_val, 'y': y_val}\n        return train_set, val_set\n\n    def get_params(self):\n        \"\"\"\n        for now stolen from https://github.com/ghmagazine/kagglebook/blob/master/ch06/ch06-03-hopt_nn.py\n        \"\"\"\n        params = {\n            'input_dropout': 0.0,\n            'hidden_layers': 2,\n            'hidden_units': 128,\n            'embedding_out_dim': 4,\n            'hidden_activation': 'relu',\n            'hidden_dropout': 0.05,\n            'batch_norm': 'before_act',\n            'optimizer': {'type': 'adam', 'lr': 0.001},\n            'batch_size': 128,\n            'epochs': 80\n        }\n\n        return params","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dict = {}\nfor i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/*'):\n    name = i.split('/')[-1].split('.')[0]\n    if name != 'WTeamSpellings':\n        data_dict[name] = pd.read_csv(i)\n    else:\n        data_dict[name] = pd.read_csv(i, encoding='cp1252')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Overview"},{"metadata":{},"cell_type":"markdown","source":"NCAAW20 has less data than NCAAM20. It may be easier for us to start with NCAAW20."},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dict.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'Cities'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeamSpellings'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WSeasons'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeams'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneyCompactResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WGameCities'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'Conferences'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneySeeds'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get int from seed\ndata_dict['WNCAATourneySeeds']['Seed'] = data_dict['WNCAATourneySeeds']['Seed'].apply(lambda x: int(x[1:3]))\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneySlots'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeamConferences'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneyDetailedResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WRegularSeasonDetailedResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WRegularSeasonCompactResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# let's also have a look at test\ntest = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# format ID\ntest = test.drop(['Pred'], axis=1)\ntest['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))\ntest['WTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[1]))\ntest['LTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[2]))\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data processing and feature engineering.\n\nThe main idea is to extract features, which could be useful to understand how much one team is better than another one."},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge tables ============\ntrain = data_dict['WNCAATourneyCompactResults'] # use compact data only for now\n\n# # compact <- detailed (Tourney files)\n# train = pd.merge(data_dict['MNCAATourneyCompactResults'], data_dict['MNCAATourneyDetailedResults'], how='left',\n#              on=['Season', 'DayNum', 'WTeamID', 'WScore', 'LTeamID', 'LScore', 'WLoc', 'NumOT'])\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train =================================\n# merge with Game Cities\ngameCities = pd.merge(data_dict['WGameCities'], data_dict['Cities'], how='left', on=['CityID'])\ncols_to_use = gameCities.columns.difference(train.columns).tolist() + [\"Season\", \"WTeamID\", \"LTeamID\"]\ntrain = train.merge(gameCities[cols_to_use], how=\"left\", on=[\"Season\", \"WTeamID\", \"LTeamID\"])\ntrain.head()\n\n# merge with WSeasons\ncols_to_use = data_dict[\"WSeasons\"].columns.difference(train.columns).tolist() + [\"Season\"]\ntrain = train.merge(data_dict[\"WSeasons\"][cols_to_use], how=\"left\", on=[\"Season\"])\ntrain.head()\n\n# merge with WTeams\ncols_to_use = data_dict[\"WTeams\"].columns.difference(train.columns).tolist()\ntrain = train.merge(data_dict[\"WTeams\"][cols_to_use], how=\"left\", left_on=[\"WTeamID\"], right_on=[\"TeamID\"])\ntrain.drop(['TeamID'], axis=1, inplace=True)\ntrain = train.merge(data_dict[\"WTeams\"][cols_to_use], how=\"left\", left_on=[\"LTeamID\"], right_on=[\"TeamID\"], suffixes=('_W', '_L'))\ntrain.drop(['TeamID'], axis=1, inplace=True)\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with WNCAATourneySeeds\ncols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(train.columns).tolist() + ['Season']\ntrain = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                    how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])\ntrain.drop(['TeamID'], axis=1, inplace=True)\ntrain = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                    how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))\ntrain.drop(['TeamID'], axis=1, inplace=True)\n\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test =================================\n# merge with Game Cities\ncols_to_use = gameCities.columns.difference(test.columns).tolist() + [\"Season\", \"WTeamID\", \"LTeamID\"]\ntest = test.merge(gameCities[cols_to_use].drop_duplicates(subset=[\"Season\", \"WTeamID\", \"LTeamID\"]),\n                  how=\"left\", on=[\"Season\", \"WTeamID\", \"LTeamID\"])\ndel gameCities\ngc.collect()\ntest.head()\n\n# merge with WSeasons\ncols_to_use = data_dict[\"WSeasons\"].columns.difference(test.columns).tolist() + [\"Season\"]\ntest = test.merge(data_dict[\"WSeasons\"][cols_to_use].drop_duplicates(subset=[\"Season\"]),\n                  how=\"left\", on=[\"Season\"])\ntest.head()\n\n# merge with WTeams\ncols_to_use = data_dict[\"WTeams\"].columns.difference(test.columns).tolist()\ntest = test.merge(data_dict[\"WTeams\"][cols_to_use].drop_duplicates(subset=[\"TeamID\"]),\n                  how=\"left\", left_on=[\"WTeamID\"], right_on=[\"TeamID\"])\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest = test.merge(data_dict[\"WTeams\"][cols_to_use].drop_duplicates(subset=[\"TeamID\"]),\n                  how=\"left\", left_on=[\"LTeamID\"], right_on=[\"TeamID\"], suffixes=('_W', '_L'))\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest.head()\n\n# merge with WNCAATourneySeeds\ncols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(test.columns).tolist() + ['Season']\ntest = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                  how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                  how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))\ntest.drop(['TeamID'], axis=1, inplace=True)\n\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist()]\nprint(not_exist_in_test)\ntrain = train.drop(not_exist_in_test, axis=1)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# compact <- detailed (regular season files)\nregularSeason = data_dict['WRegularSeasonCompactResults']\n# regularSeason = pd.merge(data_dict['WRegularSeasonCompactResults'], data_dict['WRegularSeasonDetailedResults'], how='left',\n#              on=['Season', 'DayNum', 'WTeamID', 'WScore', 'LTeamID', 'LScore', 'WLoc', 'NumOT'])\nprint(regularSeason.shape)\nregularSeason.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split winners and losers\nteam_win_score = regularSeason.groupby(['Season', 'WTeamID']).agg({'WScore':['sum', 'count', 'var']}).reset_index()\nteam_win_score.columns = [' '.join(col).strip() for col in team_win_score.columns.values]\nteam_loss_score = regularSeason.groupby(['Season', 'LTeamID']).agg({'LScore':['sum', 'count', 'var']}).reset_index()\nteam_loss_score.columns = [' '.join(col).strip() for col in team_loss_score.columns.values]\ndel regularSeason\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(team_win_score.shape)\nteam_win_score.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(team_loss_score.shape)\nteam_loss_score.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with train \ntrain = pd.merge(train, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])\ntrain = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])\ntrain = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])\ntrain = pd.merge(train, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])\ntrain.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with test \ntest = pd.merge(test, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])\ntest = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])\ntest = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])\ntest = pd.merge(test, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])\ntest.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preprocess\ndef preprocess(df):\n    df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']\n    df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']\n    df['x_count'] = df['WScore count_x'] + df['LScore count_y']\n    df['y_count'] = df['WScore count_y'] + df['WScore count_x']\n    df['x_var'] = df['WScore var_x'] + df['LScore count_y']\n    df['y_var'] = df['WScore var_y'] + df['WScore var_x']\n    return df\ntrain = preprocess(train)\ntest = preprocess(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make winner and loser train\ntrain_win = train.copy()\ntrain_los = train.copy()\ntrain_win = train_win[['Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L', \n                 'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntrain_los = train_los[['Seed_L', 'Seed_W', 'TeamName_L', 'TeamName_W', \n                 'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntrain_win.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',\n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']\ntrain_los.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',\n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']\n\n# same processing for test\ntest = test[['ID', 'Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L', \n                 'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntest.columns = ['ID', 'Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',\n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature enginnering\ndef feature_engineering(df):\n    df['Seed_diff'] = df['Seed_1'] - df['Seed_2']\n    df['Score_diff'] = df['Score_1'] - df['Score_2']\n    df['Count_diff'] = df['Count_1'] - df['Count_2']\n    df['Var_diff'] = df['Var_1'] - df['Var_2']\n    df['Mean_score1'] = df['Score_1'] / df['Count_1']\n    df['Mean_score2'] = df['Score_2'] / df['Count_2']\n    df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']\n    df['FanoFactor_1'] = df['Var_1'] / df['Mean_score1']\n    df['FanoFactor_2'] = df['Var_2'] / df['Mean_score2']\n    return df\ntrain_win = feature_engineering(train_win)\ntrain_los = feature_engineering(train_los)\ntest = feature_engineering(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_win[\"result\"] = 1\nprint(train_win.shape)\ntrain_win.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_los[\"result\"] = 0\nprint(train_los.shape)\ntrain_los.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.concat((train_win, train_los)).reset_index(drop=True)\nprint(data.shape)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# label encoding\ncategoricals = [\"TeamName_1\", \"TeamName_2\"]\nfor c in categoricals:\n    le = LabelEncoder()\n    data[c] = data[c].fillna(\"NaN\")\n    data[c] = le.fit_transform(data[c])\n    test[c] = le.transform(test[c])\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict & Make Submission File"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'result'\nfeatures = data.columns.values.tolist()\nfeatures.remove(target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fit MLP"},{"metadata":{"trusted":true},"cell_type":"code","source":"nn = NeuralNetworkModel(data, test, target, features, categoricals=categoricals, n_splits=10, \n                cv_method=\"StratifiedKFold\", group=None, task=\"classification\", scaler=\"MinMax\", verbose=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fit LGB"},{"metadata":{"trusted":true},"cell_type":"code","source":"lgbm = LgbModel(data, test, target, features, categoricals=categoricals, n_splits=10, \n                cv_method=\"StratifiedKFold\", group=None, task=\"classification\", scaler=None, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature importance\nlgbm.plot_feature_importance()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fit CatBoost"},{"metadata":{"trusted":true},"cell_type":"code","source":"catb = CatbModel(data, test, target, features, categoricals=categoricals, n_splits=10,\n                cv_method=\"StratifiedKFold\", group=None, task=\"classification\", scaler=None, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature importance\ncatb.plot_feature_importance()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = 0.7 * lgbm.y_pred + 0.2 * catb.y_pred + 0.1 * nn.y_pred\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}