{"cells":[{"metadata":{},"cell_type":"markdown","source":"Base on:  \n* https://www.kaggle.com/braquino/convert-to-regression  \n* https://www.kaggle.com/ratan123/march-madness-2020-ncaam-simple-lightgbm-on-kfold  \n\nWith XGB and LGB Blending.  \nIf it helps,  \nPlease help upvote this notebook and the original one, thanks."},{"metadata":{},"cell_type":"markdown","source":"### Import libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GridSearchCV\nimport matplotlib.pyplot as plt\nfrom matplotlib import pyplot\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom xgboost import XGBClassifier, XGBRegressor\nfrom xgboost import plot_importance\nfrom catboost import CatBoostClassifier, CatBoostRegressor\nimport shap\n\nfrom time import time\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.metrics import log_loss, accuracy_score\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn.preprocessing import OneHotEncoder\nimport gc\nimport json\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def read_data():\n    tourney_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')\n    tourney_seed = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')\n    season_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')\n    test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\n    submission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\n    return tourney_result, tourney_seed, season_result, test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_test(tourney_result,tourney_seed,season_result,test_df):\n    # deleting unnecessary columns\n    tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)\n    # Merge Seed\n    tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\n    tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)\n    tourney_result = tourney_result.drop('TeamID', axis=1)\n    tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\n    tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)\n    tourney_result = tourney_result.drop('TeamID', axis=1)\n\n    def get_seed(x):\n        return int(x[1:3])\n\n    tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))\n    tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))\n    # Merge Score\n    season_win_result = season_result[['Season', 'WTeamID', 'WScore']]\n    season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]\n    season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)\n    season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)\n    season_result = pd.concat((season_win_result, season_lose_result)).reset_index(drop=True)\n    season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()\n    tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\n    tourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)\n    tourney_result = tourney_result.drop('TeamID', axis=1)\n    tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\n    tourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)\n    tourney_result = tourney_result.drop('TeamID', axis=1)\n    tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)\n    tourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True)\n    tourney_lose_result = tourney_win_result.copy()\n    tourney_lose_result['Seed1'] = tourney_win_result['Seed2']\n    tourney_lose_result['Seed2'] = tourney_win_result['Seed1']\n    tourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']\n    tourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']\n    tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']\n    tourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']\n    tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']\n    tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']\n    tourney_win_result['result'] = 1\n    tourney_lose_result['result'] = 0\n    tourney_result = pd.concat((tourney_win_result, tourney_lose_result)).reset_index(drop=True)\n    train_df = tourney_result\n    # Get Test\n    test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))\n    test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))\n    test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))\n    test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\n    test_df.rename(columns={'Seed':'Seed1'}, inplace=True)\n    test_df = test_df.drop('TeamID', axis=1)\n    test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\n    test_df.rename(columns={'Seed':'Seed2'}, inplace=True)\n    test_df = test_df.drop('TeamID', axis=1)\n    test_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\n    test_df.rename(columns={'Score':'ScoreT1'}, inplace=True)\n    test_df = test_df.drop('TeamID', axis=1)\n    test_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\n    test_df.rename(columns={'Score':'ScoreT2'}, inplace=True)\n    test_df = test_df.drop('TeamID', axis=1)\n    test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))\n    test_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))\n    test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']\n    test_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']\n    test_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)\n    return train_df, test_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_result, tourney_seed, season_result, test_df = read_data()\ntrain_df, test_df = get_train_test(tourney_result,tourney_seed,season_result,test_df)\ntest_df['result']=np.NaN\ndel tourney_result, tourney_seed, season_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Train dataset has {train_df.shape[0]} rows and {train_df.shape[1]} cols\")\nprint(f\"Test dataset has {test_df.shape[0]} rows and {test_df.shape[1]} cols\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model training"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Base_Model(object):\n    \n    def __init__(self, train_df, test_df, features, categoricals=[], n_splits=5, verbose=True):\n        self.train_df = train_df\n        self.test_df = test_df\n        self.features = features\n        self.n_splits = n_splits\n        self.categoricals = categoricals\n        self.target = 'result'\n        self.cv = self.get_cv()\n        self.verbose = verbose\n        self.params = self.get_params()\n        self.y_pred, self.model = self.fit()\n        \n    def train_model(self, train_set, val_set):\n        raise NotImplementedError\n        \n    def get_cv(self):\n        cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)\n        return cv.split(self.train_df, self.train_df[self.target])\n    \n    def get_params(self):\n        raise NotImplementedError\n        \n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        raise NotImplementedError\n        \n    def convert_x(self, x):\n        return x\n        \n    def fit(self):\n        oof_pred = np.zeros((len(train_df), ))\n        y_pred = np.zeros((len(test_df), ))\n        for fold, (train_idx, val_idx) in enumerate(self.cv):\n            print('Fold:',fold+1)\n            x_train, x_val = self.train_df[self.features].iloc[train_idx], self.train_df[self.features].iloc[val_idx]\n            y_train, y_val = self.train_df[self.target][train_idx], self.train_df[self.target][val_idx]\n            train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)\n            model = self.train_model(train_set, val_set)\n            \n            conv_x_val = self.convert_x(x_val)\n            oof_pred[val_idx] = model.predict(conv_x_val).reshape(oof_pred[val_idx].shape)\n            \n            x_test = self.convert_x(self.test_df[self.features])\n            y_pred += model.predict(x_test).reshape(y_pred.shape) / self.n_splits\n        return y_pred, model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Lgb_Model(Base_Model):\n    \n    def train_model(self, train_set, val_set):\n        verbosity = 100 if self.verbose else 0\n        return lgb.train(self.params, train_set, 10000, valid_sets=[train_set, val_set], verbose_eval=verbosity)\n        \n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)\n        val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)\n        return train_set, val_set\n        \n    def get_params(self):\n        params = {'num_leaves': 400,\n                  'min_child_weight': 0.034,\n                  'feature_fraction': 0.379,\n                  'bagging_fraction': 0.418,\n                  'min_data_in_leaf': 106,\n                  'objective': 'binary',\n                  'max_depth': -1,\n                  'learning_rate': 0.0068,\n                  \"boosting_type\": \"gbdt\",\n                  \"bagging_seed\": 11,\n                  \"metric\": 'logloss',\n                  \"verbosity\": -1,\n                  'reg_alpha': 0.3899,\n                  'reg_lambda': 0.648,\n                  'random_state': 47,\n                    }\n        return params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Xgb_Model(Base_Model):\n    \n    def train_model(self, train_set, val_set):\n        verbosity = 100 if self.verbose else 0\n        return xgb.train(self.params, train_set, \n                         num_boost_round=5000, evals=[(train_set, 'train'), (val_set, 'val')], \n                         verbose_eval=verbosity, early_stopping_rounds=100)\n        \n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        train_set = xgb.DMatrix(x_train, y_train)\n        val_set = xgb.DMatrix(x_val, y_val)\n        return train_set, val_set\n    \n    def convert_x(self, x):\n        return xgb.DMatrix(x)\n        \n    def get_params(self):\n        params = { 'colsample_bytree': 0.8,                 \n                   'learning_rate': 0.01,\n                   'max_depth': 3,\n                   'subsample': 1,\n                   'objective':'binary:logistic',\n                   'eval_metric':'logloss',\n                   'min_child_weight':3,\n                   'gamma':0.25,\n                   'n_estimators':5000}\n        return params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Catb_Model(Base_Model):\n    \n    def train_model(self, train_df, test_df):\n        verbosity = 100 if self.verbose else 0\n        clf = CatBoostClassifier(**self.params)\n        clf.fit(train_df['X'], \n                train_df['y'], \n                eval_set=(test_df['X'], test_df['y']),\n                verbose=verbosity, \n                cat_features=self.categoricals)\n        return clf\n        \n    def convert_dataset(self, x_train, y_train, x_val, y_val):\n        train_set = {'X': x_train, 'y': y_train}\n        val_set = {'X': x_val, 'y': y_val}\n        return train_set, val_set\n        \n    def get_params(self):\n        params = {'loss_function': 'Logloss',\n                   'task_type': \"CPU\",\n                   'iterations': 5000,\n                   'od_type': \"Iter\",\n                    'depth': 3,\n                  'colsample_bylevel': 0.5, \n                   'early_stopping_rounds': 300,\n                    'l2_leaf_reg': 18,\n                   'random_seed': 42,\n                    'use_best_model': True\n                    }\n        return params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = train_df.columns\nfeatures = [x for x in features if x not in ['result']]\nprint(features)\ncategoricals = []\n\n#cat_model = Catb_Model(train_df, test_df, features, categoricals=categoricals)\nlgb_model = Lgb_Model(train_df, test_df, features, categoricals=categoricals)\nxgb_model = Xgb_Model(train_df, test_df, features, categoricals=categoricals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = {'lgb': 0.60, 'cat':0, 'xgb':0.40}\nsubmission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = (lgb_model.y_pred*weights['lgb']) + (xgb_model.y_pred*weights['xgb'])\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}