{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NCAAM2020: XGBoost + LightGBM K-Fold Ensemble (Baseline)","metadata":{}},{"cell_type":"markdown","source":"- Original logs from 2020:\n    - Built upon from https://www.kaggle.com/ratan123/march-madness-2020-ncaam-simple-lightgbm-on-kfold\n    - Added XGBoost and K-fold ensemble\n- 2021 logs:\n    - added option to use 2021 data in stage 1","metadata":{}},{"cell_type":"markdown","source":"## Master Configs","metadata":{}},{"cell_type":"code","source":"YEAR = 2021 # or 2020 from google cloud competition\nSTAGE = 2\nNCAAM = True\nNCAAW = False","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:11.364323Z","iopub.execute_input":"2021-09-21T07:46:11.365057Z","iopub.status.idle":"2021-09-21T07:46:11.371005Z","shell.execute_reply.started":"2021-09-21T07:46:11.364985Z","shell.execute_reply":"2021-09-21T07:46:11.370007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if NCAAM:\n    DATA_DIR = f'../input/ncaam-march-mania-2021/MDataFiles_Stage{STAGE}/'\nif NCAAW:\n    DATA_DIR = f'../input/ncaaw-march-mania-2021/WDataFiles_Stage{STAGE}/'","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:11.373429Z","iopub.execute_input":"2021-09-21T07:46:11.373975Z","iopub.status.idle":"2021-09-21T07:46:11.389792Z","shell.execute_reply.started":"2021-09-21T07:46:11.373738Z","shell.execute_reply":"2021-09-21T07:46:11.388377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Library & Load Data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.linear_model import LogisticRegression\nimport matplotlib.pyplot as plt\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import accuracy_score\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom xgboost import XGBClassifier\nimport gc","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:11.391923Z","iopub.execute_input":"2021-09-21T07:46:11.392325Z","iopub.status.idle":"2021-09-21T07:46:13.306809Z","shell.execute_reply.started":"2021-09-21T07:46:11.39226Z","shell.execute_reply":"2021-09-21T07:46:13.305922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if YEAR == 2020:\n    tourney_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')\n    tourney_seed = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')\nelif YEAR == 2021:\n    tourney_result = pd.read_csv(DATA_DIR + 'MNCAATourneyCompactResults.csv')\n    tourney_seed = pd.read_csv(DATA_DIR + 'MNCAATourneySeeds.csv')","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-09-21T07:46:13.308802Z","iopub.execute_input":"2021-09-21T07:46:13.30906Z","iopub.status.idle":"2021-09-21T07:46:13.35123Z","shell.execute_reply.started":"2021-09-21T07:46:13.309016Z","shell.execute_reply":"2021-09-21T07:46:13.35029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# deleting unnecessary columns\ntourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)\ntourney_result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.35402Z","iopub.execute_input":"2021-09-21T07:46:13.354322Z","iopub.status.idle":"2021-09-21T07:46:13.384974Z","shell.execute_reply.started":"2021-09-21T07:46:13.354274Z","shell.execute_reply":"2021-09-21T07:46:13.384127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge Seed","metadata":{}},{"cell_type":"code","source":"tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.386183Z","iopub.execute_input":"2021-09-21T07:46:13.386427Z","iopub.status.idle":"2021-09-21T07:46:13.430862Z","shell.execute_reply.started":"2021-09-21T07:46:13.386387Z","shell.execute_reply":"2021-09-21T07:46:13.429873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_seed(x):\n    return int(x[1:3])\n\ntourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))\ntourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))\ntourney_result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.432601Z","iopub.execute_input":"2021-09-21T07:46:13.432962Z","iopub.status.idle":"2021-09-21T07:46:13.460041Z","shell.execute_reply.started":"2021-09-21T07:46:13.432894Z","shell.execute_reply":"2021-09-21T07:46:13.458909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge Score","metadata":{}},{"cell_type":"code","source":"if YEAR == 2020:\n    season_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')\nelif YEAR == 2021:\n    season_result = pd.read_csv(DATA_DIR + 'MRegularSeasonCompactResults.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.464121Z","iopub.execute_input":"2021-09-21T07:46:13.464397Z","iopub.status.idle":"2021-09-21T07:46:13.636223Z","shell.execute_reply.started":"2021-09-21T07:46:13.464353Z","shell.execute_reply":"2021-09-21T07:46:13.635344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"season_win_result = season_result[['Season', 'WTeamID', 'WScore']]\nseason_lose_result = season_result[['Season', 'LTeamID', 'LScore']]\nseason_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)\nseason_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)\nseason_result = pd.concat((season_win_result, season_lose_result)).reset_index(drop=True)\nseason_result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.638277Z","iopub.execute_input":"2021-09-21T07:46:13.638551Z","iopub.status.idle":"2021-09-21T07:46:13.691177Z","shell.execute_reply.started":"2021-09-21T07:46:13.638514Z","shell.execute_reply":"2021-09-21T07:46:13.690217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()\nseason_score","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.692418Z","iopub.execute_input":"2021-09-21T07:46:13.692662Z","iopub.status.idle":"2021-09-21T07:46:13.743117Z","shell.execute_reply.started":"2021-09-21T07:46:13.692628Z","shell.execute_reply":"2021-09-21T07:46:13.742307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.744585Z","iopub.execute_input":"2021-09-21T07:46:13.744935Z","iopub.status.idle":"2021-09-21T07:46:13.782842Z","shell.execute_reply.started":"2021-09-21T07:46:13.744879Z","shell.execute_reply":"2021-09-21T07:46:13.781847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)\ntourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True)\ntourney_win_result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.784305Z","iopub.execute_input":"2021-09-21T07:46:13.784601Z","iopub.status.idle":"2021-09-21T07:46:13.80067Z","shell.execute_reply.started":"2021-09-21T07:46:13.784555Z","shell.execute_reply":"2021-09-21T07:46:13.799633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tourney_lose_result = tourney_win_result.copy()\ntourney_lose_result['Seed1'] = tourney_win_result['Seed2']\ntourney_lose_result['Seed2'] = tourney_win_result['Seed1']\ntourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']\ntourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']\ntourney_lose_result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.80209Z","iopub.execute_input":"2021-09-21T07:46:13.802523Z","iopub.status.idle":"2021-09-21T07:46:13.825403Z","shell.execute_reply.started":"2021-09-21T07:46:13.80247Z","shell.execute_reply":"2021-09-21T07:46:13.824401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Training Data","metadata":{}},{"cell_type":"code","source":"tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']\ntourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']\ntourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']\ntourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.82668Z","iopub.execute_input":"2021-09-21T07:46:13.826926Z","iopub.status.idle":"2021-09-21T07:46:13.936891Z","shell.execute_reply.started":"2021-09-21T07:46:13.826885Z","shell.execute_reply":"2021-09-21T07:46:13.936186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tourney_win_result['result'] = 1\ntourney_lose_result['result'] = 0\ntourney_result = pd.concat((tourney_win_result, tourney_lose_result)).reset_index(drop=True)\ntourney_result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.938018Z","iopub.execute_input":"2021-09-21T07:46:13.93847Z","iopub.status.idle":"2021-09-21T07:46:13.958227Z","shell.execute_reply.started":"2021-09-21T07:46:13.938412Z","shell.execute_reply":"2021-09-21T07:46:13.957567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing testing data","metadata":{}},{"cell_type":"code","source":"if YEAR == 2020:\n    test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\nelif YEAR == 2021:\n    test_df = pd.read_csv(DATA_DIR + f'MSampleSubmissionStage{STAGE}.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.959345Z","iopub.execute_input":"2021-09-21T07:46:13.959756Z","iopub.status.idle":"2021-09-21T07:46:13.976675Z","shell.execute_reply.started":"2021-09-21T07:46:13.959708Z","shell.execute_reply":"2021-09-21T07:46:13.975305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))\ntest_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))\ntest_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:13.978398Z","iopub.execute_input":"2021-09-21T07:46:13.9787Z","iopub.status.idle":"2021-09-21T07:46:14.011505Z","shell.execute_reply.started":"2021-09-21T07:46:13.978644Z","shell.execute_reply":"2021-09-21T07:46:14.010469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Seed':'Seed1'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Seed':'Seed2'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Score':'ScoreT1'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Score':'ScoreT2'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:14.012991Z","iopub.execute_input":"2021-09-21T07:46:14.013296Z","iopub.status.idle":"2021-09-21T07:46:14.079893Z","shell.execute_reply.started":"2021-09-21T07:46:14.013241Z","shell.execute_reply":"2021-09-21T07:46:14.079253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))\ntest_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))\ntest_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']\ntest_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']\ntest_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:14.081002Z","iopub.execute_input":"2021-09-21T07:46:14.08144Z","iopub.status.idle":"2021-09-21T07:46:14.109773Z","shell.execute_reply.started":"2021-09-21T07:46:14.081385Z","shell.execute_reply":"2021-09-21T07:46:14.109162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"X = tourney_result.drop('result', axis=1)\ny = tourney_result.result","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:46:14.110894Z","iopub.execute_input":"2021-09-21T07:46:14.111304Z","iopub.status.idle":"2021-09-21T07:46:14.116509Z","shell.execute_reply.started":"2021-09-21T07:46:14.111259Z","shell.execute_reply":"2021-09-21T07:46:14.11562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Training & Evaluation","metadata":{}},{"cell_type":"markdown","source":"### LightGBM Predictions","metadata":{}},{"cell_type":"code","source":"NFOLDS = 10\nfolds = KFold(n_splits=NFOLDS)\n\ncolumns = X.columns\nsplits = folds.split(X, y)\ny_preds_lgb = np.zeros(test_df.shape[0])\n  \nfor fold_n, (train_index, valid_index) in enumerate(splits):\n    print('Fold:', fold_n + 1)\n    X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n    \n    dtrain = lgb.Dataset(X_train, label=y_train)\n    dvalid = lgb.Dataset(X_valid, label=y_valid)\n    \n    clf = lgb.LGBMClassifier(boosting_type=\"gbdt\", \n                             num_leaves=127,\n                             max_depth=-1,\n                             learning_rate=0.01, \n                             n_estimators=10000,\n                             min_child_samples=10,\n                             objective=\"binary\")\n    clf.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], verbose=5000)\n    \n    y_preds_lgb += clf.predict(test_df) / NFOLDS\n    \n    del X_train, X_valid, y_train, y_valid\n    gc.collect()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-09-21T07:46:14.118282Z","iopub.execute_input":"2021-09-21T07:46:14.118809Z","iopub.status.idle":"2021-09-21T07:53:33.749395Z","shell.execute_reply.started":"2021-09-21T07:46:14.11875Z","shell.execute_reply":"2021-09-21T07:53:33.748489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XGBoost Predictions","metadata":{}},{"cell_type":"code","source":"y_preds_lgb","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:53:33.750958Z","iopub.execute_input":"2021-09-21T07:53:33.751339Z","iopub.status.idle":"2021-09-21T07:53:33.757668Z","shell.execute_reply.started":"2021-09-21T07:53:33.75127Z","shell.execute_reply":"2021-09-21T07:53:33.756755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble & Make Submission File","metadata":{}},{"cell_type":"code","source":"if YEAR == 2020:\n    submission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\nelif YEAR == 2021:\n    submission_df = pd.read_csv(DATA_DIR + f'MSampleSubmissionStage{STAGE}.csv')\nsubmission_df['Pred'] = y_preds_lgb\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:53:33.759476Z","iopub.execute_input":"2021-09-21T07:53:33.759944Z","iopub.status.idle":"2021-09-21T07:53:33.792327Z","shell.execute_reply.started":"2021-09-21T07:53:33.759871Z","shell.execute_reply":"2021-09-21T07:53:33.791388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df['Pred'].hist()","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:53:33.794203Z","iopub.execute_input":"2021-09-21T07:53:33.794583Z","iopub.status.idle":"2021-09-21T07:53:34.088788Z","shell.execute_reply.started":"2021-09-21T07:53:33.794514Z","shell.execute_reply":"2021-09-21T07:53:34.087618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-21T07:53:34.090502Z","iopub.execute_input":"2021-09-21T07:53:34.091182Z","iopub.status.idle":"2021-09-21T07:53:34.683795Z","shell.execute_reply.started":"2021-09-21T07:53:34.091053Z","shell.execute_reply":"2021-09-21T07:53:34.682896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Future Work\n\n- GridSearchCV (since this runs considerably fast)\n- CatBoost\n- LSTM\n- Ensembles of XGBoost, LightGBM, CatBoost, LSTM\n- Ensemble weights by correlations","metadata":{}}]}