{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#importing libraries\nimport matplotlib.pyplot as plt\nimport seaborn as sns\npd.set_option('max_columns', None)\nplt.style.use('fivethirtyeight')\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder\n%matplotlib inline\nimport copy\nimport datetime\nfrom sklearn.utils import shuffle\nfrom scipy import stats\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold, cross_val_score, GridSearchCV, RepeatedStratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport lightgbm as lgb\nimport optuna\nfrom optuna.visualization import plot_optimization_history\nfrom sklearn import model_selection\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, roc_auc_score, log_loss, classification_report, confusion_matrix\nimport json\nimport ast\nimport time\nfrom sklearn import linear_model\n\nimport math\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport glob\nimport gc\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import LabelEncoder\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Loading data\ndata_dict = {}\nfor i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/*'):\n    name = i.split('/')[-1].split('.')[0]\n    if name != 'WTeamSpellings':\n        data_dict[name] = pd.read_csv(i)\n    else:\n        data_dict[name] = pd.read_csv(i, encoding='cp1252')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Data Overview**"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dict.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'Cities'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeamSpellings'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WSeasons'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeams'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneyCompactResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WGameCities'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'Conferences'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneySeeds'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get int from seed\ndata_dict['WNCAATourneySeeds']['Seed'] = data_dict['WNCAATourneySeeds']['Seed'].apply(lambda x: int(x[1:3]))\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneySlots'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WTeamConferences'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WNCAATourneyDetailedResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WRegularSeasonDetailedResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = 'WRegularSeasonCompactResults'\nprint(data_dict[fname].shape)\ndata_dict[fname].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# let's also have a look at test\ntest = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# formatting ID\ntest = test.drop(['Pred'], axis=1)\ntest['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))\ntest['WTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[1]))\ntest['LTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[2]))\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data processing and feature engineering.\nThe main idea is to extract features, which could be useful to understand how much one team is better than another one."},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge tables ============\ntrain = data_dict['WNCAATourneyCompactResults']\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train =================================\n# merge with Game Cities\ngameCities = pd.merge(data_dict['WGameCities'], data_dict['Cities'], how='left', on=['CityID'])\ncols_to_use = gameCities.columns.difference(train.columns).tolist() + [\"Season\", \"WTeamID\", \"LTeamID\"]\ntrain = train.merge(gameCities[cols_to_use], how=\"left\", on=[\"Season\", \"WTeamID\", \"LTeamID\"])\ntrain.head()\n\n# merge with WSeasons\ncols_to_use = data_dict[\"WSeasons\"].columns.difference(train.columns).tolist() + [\"Season\"]\ntrain = train.merge(data_dict[\"WSeasons\"][cols_to_use], how=\"left\", on=[\"Season\"])\ntrain.head()\n\n# merge with WTeams\ncols_to_use = data_dict[\"WTeams\"].columns.difference(train.columns).tolist()\ntrain = train.merge(data_dict[\"WTeams\"][cols_to_use], how=\"left\", left_on=[\"WTeamID\"], right_on=[\"TeamID\"])\ntrain.drop(['TeamID'], axis=1, inplace=True)\ntrain = train.merge(data_dict[\"WTeams\"][cols_to_use], how=\"left\", left_on=[\"LTeamID\"], right_on=[\"TeamID\"], suffixes=('_W', '_L'))\ntrain.drop(['TeamID'], axis=1, inplace=True)\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with WNCAATourneySeeds\ncols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(train.columns).tolist() + ['Season']\ntrain = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                    how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])\ntrain.drop(['TeamID'], axis=1, inplace=True)\ntrain = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                    how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))\ntrain.drop(['TeamID'], axis=1, inplace=True)\n\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with Game Cities\ncols_to_use = gameCities.columns.difference(test.columns).tolist() + [\"Season\", \"WTeamID\", \"LTeamID\"]\ntest = test.merge(gameCities[cols_to_use].drop_duplicates(subset=[\"Season\", \"WTeamID\", \"LTeamID\"]),\n                  how=\"left\", on=[\"Season\", \"WTeamID\", \"LTeamID\"])\ndel gameCities\ngc.collect()\ntest.head()\n\n# merge with WSeasons\ncols_to_use = data_dict[\"WSeasons\"].columns.difference(test.columns).tolist() + [\"Season\"]\ntest = test.merge(data_dict[\"WSeasons\"][cols_to_use].drop_duplicates(subset=[\"Season\"]),\n                  how=\"left\", on=[\"Season\"])\ntest.head()\n\n# merge with WTeams\ncols_to_use = data_dict[\"WTeams\"].columns.difference(test.columns).tolist()\ntest = test.merge(data_dict[\"WTeams\"][cols_to_use].drop_duplicates(subset=[\"TeamID\"]),\n                  how=\"left\", left_on=[\"WTeamID\"], right_on=[\"TeamID\"])\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest = test.merge(data_dict[\"WTeams\"][cols_to_use].drop_duplicates(subset=[\"TeamID\"]),\n                  how=\"left\", left_on=[\"LTeamID\"], right_on=[\"TeamID\"], suffixes=('_W', '_L'))\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest.head()\n\n# merge with WNCAATourneySeeds\ncols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(test.columns).tolist() + ['Season']\ntest = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                  how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])\ntest.drop(['TeamID'], axis=1, inplace=True)\ntest = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=[\"Season\",\"TeamID\"]),\n                  how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))\ntest.drop(['TeamID'], axis=1, inplace=True)\n\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist()]\nprint(not_exist_in_test)\ntrain = train.drop(not_exist_in_test, axis=1)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"regularSeason = data_dict['WRegularSeasonCompactResults']\nprint(regularSeason.shape)\nregularSeason.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split winners and losers\nteam_win_score = regularSeason.groupby(['Season', 'WTeamID']).agg({'WScore':['sum', 'count', 'var']}).reset_index()\nteam_win_score.columns = [' '.join(col).strip() for col in team_win_score.columns.values]\nteam_loss_score = regularSeason.groupby(['Season', 'LTeamID']).agg({'LScore':['sum', 'count', 'var']}).reset_index()\nteam_loss_score.columns = [' '.join(col).strip() for col in team_loss_score.columns.values]\ndel regularSeason\ngc.collect()\nprint(team_win_score.shape)\nteam_win_score.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(team_loss_score.shape)\nteam_loss_score.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with train \ntrain = pd.merge(train, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])\ntrain = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])\ntrain = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])\ntrain = pd.merge(train, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])\ntrain.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with test \ntest = pd.merge(test, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])\ntest = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])\ntest = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])\ntest = pd.merge(test, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])\ntest.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(df):\n    df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']\n    df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']\n    df['x_count'] = df['WScore count_x'] + df['LScore count_y']\n    df['y_count'] = df['WScore count_y'] + df['WScore count_x']\n    df['x_var'] = df['WScore var_x'] + df['LScore count_y']\n    df['y_var'] = df['WScore var_y'] + df['WScore var_x']\n    return df\ntrain = preprocess(train)\ntest = preprocess(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make winner and loser train\ntrain_win = train.copy()\ntrain_los = train.copy()\ntrain_win = train_win[['Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L', \n                 'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntrain_los = train_los[['Seed_L', 'Seed_W', 'TeamName_L', 'TeamName_W', \n                 'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntrain_win.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',\n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']\ntrain_los.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',\n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']\n\n# same processing for test\ntest = test[['ID', 'Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L', \n                 'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntest.columns = ['ID', 'Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',\n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Feature engineering\ndef feature_engineering(df):\n    df['Seed_diff'] = df['Seed_1'] - df['Seed_2']\n    df['Score_diff'] = df['Score_1'] - df['Score_2']\n    df['Count_diff'] = df['Count_1'] - df['Count_2']\n    df['Var_diff'] = df['Var_1'] - df['Var_2']\n    df['Mean_score1'] = df['Score_1'] / df['Count_1']\n    df['Mean_score2'] = df['Score_2'] / df['Count_2']\n    df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']\n    df['FanoFactor_1'] = df['Var_1'] / df['Mean_score1']\n    df['FanoFactor_2'] = df['Var_2'] / df['Mean_score2']\n    return df\ntrain_win = feature_engineering(train_win)\ntrain_los = feature_engineering(train_los)\ntest = feature_engineering(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_win[\"result\"] = 1\nprint(train_win.shape)\ntrain_win.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_los[\"result\"] = 0\nprint(train_los.shape)\ntrain_los.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.concat((train_win, train_los)).reset_index(drop=True)\nprint(data.shape)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# label encoding\ncategoricals = [\"TeamName_1\", \"TeamName_2\"]\nfor c in categoricals:\n    le = LabelEncoder()\n    data[c] = data[c].fillna(\"NaN\")\n    data[c] = le.fit_transform(data[c])\n    test[c] = le.transform(test[c])\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\npublic_LB = [0.17343, 0.16738, 0.16657, 0.16711, 0.16761, 0.16805]\nfolds = [10, 50, 100, 250, 500, 1000]\ndf_viz = pd.DataFrame({'folds': folds, 'public_LB':public_LB})\ndf_plot = df_viz.plot(x='folds', y='public_LB')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaseModel(object):\n    \"\"\"\n    Base Model Class\n\n    \"\"\"\n\n    def __init__(self, train_df, test_df, target, features, categoricals=[], \n                n_splits=3, cv_method=\"KFold\", group=None, task=\"regression\", \n                parameter_tuning=False, scaler=None, verbose=True):\n        self.train_df = train_df\n        self.test_df = test_df\n        self.target = target\n        self.features = features\n        self.n_splits = n_splits\n        self.categoricals = categoricals\n        self.cv_method = cv_method\n        self.group = group\n        self.task = task\n        self.parameter_tuning = parameter_tuning\n        self.scaler = scaler\n        self.cv = self.get_cv()\n        self.verbose = verbose\n        self.params = self.get_params()\n        self.y_pred, self.score, self.model, self.oof, self.y_val, self.fi_df = self.fit()\n\n    def train_model(self, train_set, val_set):\n        raise NotImplementedError\n\n    def get_params(self):\n        raise NotImplementedError\n\n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        raise NotImplementedError\n\n    def convert_x(self, x):\n        return x\n\n    def calc_metric(self, y_true, y_pred): # this may need to be changed based on the metric of interest\n        if self.task == \"classification\":\n            return log_loss(y_true, y_pred)\n        elif self.task == \"regression\":\n            return np.sqrt(mean_squared_error(y_true, y_pred))\n\n    def get_cv(self):\n        if self.cv_method == \"KFold\":\n            cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n            return cv.split(self.train_df)\n        elif self.cv_method == \"StratifiedKFold\":\n            cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n            return cv.split(self.train_df, self.train_df[self.target])\n        elif self.cv_method == \"TimeSeriesSplit\":\n            cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)\n            return cv.split(self.train_df)\n        elif self.cv_method == \"GroupKFold\":\n            cv = GroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n            return cv.split(self.train_df, self.train_df[self.target], self.group)\n        elif self.cv_method == \"StratifiedGroupKFold\":\n            cv = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n            return cv.split(self.train_df, self.train_df[self.target], self.group)\n\n    def fit(self):\n        # initialize\n        oof_pred = np.zeros((self.train_df.shape[0], ))\n        y_vals = np.zeros((self.train_df.shape[0], ))\n        y_pred = np.zeros((self.test_df.shape[0], ))\n        if self.group is not None:\n            if self.group in self.features:\n                self.features.remove(self.group)\n            if self.group in self.categoricals:\n                self.categoricals.remove(self.group)\n        fi = np.zeros((self.n_splits, len(self.features)))\n\n        # scaling, if necessary\n        if self.scaler is not None:\n            # fill NaN\n            numerical_features = [f for f in self.features if f not in self.categoricals]\n            self.train_df[numerical_features] = self.train_df[numerical_features].fillna(self.train_df[numerical_features].median())\n            self.test_df[numerical_features] = self.test_df[numerical_features].fillna(self.test_df[numerical_features].median())\n            self.train_df[self.categoricals] = self.train_df[self.categoricals].fillna(self.train_df[self.categoricals].mode().iloc[0])\n            self.test_df[self.categoricals] = self.test_df[self.categoricals].fillna(self.test_df[self.categoricals].mode().iloc[0])\n\n            # scaling\n            if self.scaler == \"MinMax\":\n                scaler = MinMaxScaler()\n            elif self.scaler == \"Standard\":\n                scaler = StandardScaler()\n            df = pd.concat([self.train_df[numerical_features], self.test_df[numerical_features]], ignore_index=True)\n            scaler.fit(df[numerical_features])\n            x_test = self.test_df.copy()\n            x_test[numerical_features] = scaler.transform(x_test[numerical_features])\n            x_test = [np.absolute(x_test[i]) for i in self.categoricals] + [x_test[numerical_features]]\n        else:\n            x_test = self.test_df[self.features]\n            \n        # fitting with out of fold\n        for fold, (train_idx, val_idx) in enumerate(self.cv):\n            # train test split\n            x_train, x_val = self.train_df.loc[train_idx, self.features], self.train_df.loc[val_idx, self.features]\n            y_train, y_val = self.train_df.loc[train_idx, self.target], self.train_df.loc[val_idx, self.target]\n\n            # fitting & get feature importance\n            if self.scaler is not None:\n                x_train[numerical_features] = scaler.transform(x_train[numerical_features])\n                x_val[numerical_features] = scaler.transform(x_val[numerical_features])\n                x_train = [np.absolute(x_train[i]) for i in self.categoricals] + [x_train[numerical_features]]\n                x_val = [np.absolute(x_val[i]) for i in self.categoricals] + [x_val[numerical_features]]\n            train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)\n            model, importance = self.train_model(train_set, val_set)\n            fi[fold, :] = importance\n            conv_x_val = self.convert_x(x_val)\n            y_vals[val_idx] = y_val\n            oof_pred[val_idx] = model.predict(conv_x_val).reshape(oof_pred[val_idx].shape)\n            x_test = self.convert_x(x_test)\n            y_pred += model.predict(x_test).reshape(y_pred.shape) / self.n_splits\n            print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))\n\n        # feature importance data frame\n        fi_df = pd.DataFrame()\n        for n in np.arange(self.n_splits):\n            tmp = pd.DataFrame()\n            tmp[\"features\"] = self.features\n            tmp[\"importance\"] = fi[n, :]\n            tmp[\"fold\"] = n\n            fi_df = pd.concat([fi_df, tmp], ignore_index=True)\n        gfi = fi_df[[\"features\", \"importance\"]].groupby([\"features\"]).mean().reset_index()\n        fi_df = fi_df.merge(gfi, on=\"features\", how=\"left\", suffixes=('', '_mean'))\n\n        # outputs\n        loss_score = self.calc_metric(self.train_df[self.target], oof_pred)\n        if self.verbose:\n            print('Our oof loss score is: ', loss_score)\n        return y_pred, loss_score, model, oof_pred, y_vals, fi_df\n\n    def plot_feature_importance(self, rank_range=[1, 50]):\n        # plot\n        fig, ax = plt.subplots(1, 1, figsize=(10, 20))\n        sorted_df = self.fi_df.sort_values(by = \"importance_mean\", ascending=False).reset_index().iloc[self.n_splits * (rank_range[0]-1) : self.n_splits * rank_range[1]]\n        sns.barplot(data=sorted_df, x =\"importance\", y =\"features\", orient='h')\n        ax.set_xlabel(\"feature importance\")\n        ax.spines['top'].set_visible(False)\n        ax.spines['right'].set_visible(False)\n        return sorted_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LgbModel(BaseModel):\n    \"\"\"\n    LGB wrapper\n\n    \"\"\"\n\n    def train_model(self, train_set, val_set):\n        verbosity = 100 if self.verbose else 0\n        model = lgb.train(self.params, train_set, num_boost_round = 5000, valid_sets=[train_set, val_set], verbose_eval=verbosity)\n        fi = model.feature_importance(importance_type=\"gain\")\n        return model, fi\n\n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)\n        val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)\n        return train_set, val_set\n\n    def get_params(self):\n        # params from https://www.kaggle.com/vbmokin/mm-2020-ncaam-simple-lightgbm-on-kfold-tuning\n        params = {\n          'num_leaves': 127,\n          'min_data_in_leaf': 50,\n          'max_depth': -1,\n          'learning_rate': 0.005,\n          \"boosting_type\": \"gbdt\",\n          \"bagging_seed\": 11,\n          \"verbosity\": -1,\n          'random_state': 42,\n         }\n        \n        if self.task == \"regression\":\n            params[\"objective\"] = \"regression\"\n            params[\"metric\"] = \"rmse\"\n        elif self.task == \"classification\":\n            params[\"objective\"] = \"binary\"\n            params[\"metric\"] = \"binary_logloss\"\n        \n        # Bayesian Optimization by Optuna\n        if self.parameter_tuning == True:\n            # define objective function\n            def objective(trial):\n                # train, test split\n                train_x, test_x, train_y, test_y = train_test_split(self.train_df[self.features], \n                                                                    self.train_df[self.target],\n                                                                    test_size=0.3, random_state=42)\n                dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)\n                dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)\n\n                # parameters to be explored\n                hyperparams = {'num_leaves': trial.suggest_int('num_leaves', 24, 1024),\n                        'boosting_type': 'gbdt',\n                        'objective': params[\"objective\"],\n                        'metric': params[\"metric\"],\n                        'max_depth': trial.suggest_int('max_depth', 4, 16),\n                        'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),\n                        'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),\n                        'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),\n                        'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),\n                        'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),\n                        'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),\n                        'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),\n                        'early_stopping_rounds': 100\n                        }\n\n                # LGB\n                model = lgb.train(hyperparams, dtrain, valid_sets=dtest, verbose_eval=500)\n                pred = model.predict(test_x)\n                if self.task == \"classification\":\n                    return log_loss(test_y, pred)\n                elif self.task == \"regression\":\n                    return np.sqrt(mean_squared_error(test_y, pred))\n\n            # run optimization\n            study = optuna.create_study(direction='minimize')\n            study.optimize(objective, n_trials=50)\n\n            print('Number of finished trials: {}'.format(len(study.trials)))\n            print('Best trial:')\n            trial = study.best_trial\n            print('  Value: {}'.format(trial.value))\n            print('  Params: ')\n            for key, value in trial.params.items():\n                print('    {}: {}'.format(key, value))\n\n            params = trial.params\n\n            # lower learning rate for better accuracy\n            params[\"learning_rate\"] = 0.001\n\n            # plot history\n            plot_optimization_history(study)\n\n        return params","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Predict and make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'result'\nfeatures = data.columns.values.tolist()\nfeatures.remove(target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgbm = LgbModel(data, test, target, features, categoricals=categoricals, n_splits=10, \n                cv_method=\"StratifiedKFold\", group=None, task=\"classification\", scaler=None, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgbm.plot_feature_importance()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = lgbm.y_pred\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}