{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Overview\n#### This kernel is based on last year's [Basic Starter Kernel](https://www.kaggle.com/addisonhoward/basic-starter-kernel-ncaa-men-s-dataset-2019).\n#### Adding XGB and GBM Models"},{"metadata":{},"cell_type":"markdown","source":"#### Hi everyone welcome to this kernel,In this kernel i am building XGBoost model and GBM model using H2o framework. <br>\n#### <font color='blue'>If you find this kernel useful please upvote.Also dont forget to upvote the original kernel.</font> <br>"},{"metadata":{},"cell_type":"markdown","source":"## Import Library & Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.linear_model import LogisticRegression\nimport matplotlib.pyplot as plt\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"tourney_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')\ntourney_seed = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# deleting unnecessary columns\ntourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_seed(x):\n    return int(x[1:3])\n\ntourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))\ntourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge Score"},{"metadata":{"trusted":true},"cell_type":"code","source":"season_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"season_win_result = season_result[['Season', 'WTeamID', 'WScore']]\nseason_lose_result = season_result[['Season', 'LTeamID', 'LScore']]\nseason_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)\nseason_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)\nseason_result = pd.concat((season_win_result, season_lose_result)).reset_index(drop=True)\nseason_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()\nseason_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)\ntourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True)\ntourney_win_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_lose_result = tourney_win_result.copy()\ntourney_lose_result['Seed1'] = tourney_win_result['Seed2']\ntourney_lose_result['Seed2'] = tourney_win_result['Seed1']\ntourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']\ntourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']\ntourney_lose_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']\ntourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']\ntourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']\ntourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_win_result['result'] = 1\ntourney_lose_result['result'] = 0\ntourney_result = pd.concat((tourney_win_result, tourney_lose_result)).reset_index(drop=True)\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing testing data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))\ntest_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))\ntest_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Seed':'Seed1'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Seed':'Seed2'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Score':'ScoreT1'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Score':'ScoreT2'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))\ntest_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))\ntest_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']\ntest_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']\ntest_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)\ntest_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Importing h2o framework library which runs on local host"},{"metadata":{"trusted":true},"cell_type":"code","source":"import h2o\nh2o.init(\n  nthreads=-1,            ## -1: use all available threads\n  max_mem_size = \"8G\")  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_list=list(tourney_result.columns)[:-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1=h2o.H2OFrame(tourney_result)\ntrain_1['result']=train_1['result'].asfactor()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### XGBoost Model training parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"param = {\n      \"ntrees\" : 2000\n    , \"max_depth\" : 20\n    , \"learn_rate\" : 0.02\n    , \"sample_rate\" : 0.7\n    , \"col_sample_rate_per_tree\" : 0.9\n    , \"min_rows\" : 5\n    , \"seed\": 4241\n    , \"score_tree_interval\": 100,\"stopping_metric\" :\"MSE\",\"nfolds\":8,\"fold_assignment\":\"AUTO\",\"keep_cross_validation_predictions\" : True,\"booster\":\"dart\"\n}\nfrom h2o.estimators import H2OXGBoostEstimator\nmodel_xgb = H2OXGBoostEstimator(**param)\nmodel_xgb.train(x = train_list, y = 'result', training_frame = train_1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Summary Of XGBoost Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_xgb.summary","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### GBM Model training parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"param={\n    \"ntrees\" : 1000\n    , \"max_depth\" : 20\n    , \"learn_rate\" : 0.02\n    , \"sample_rate\" : 0.7\n    , \"col_sample_rate_per_tree\" : 0.9\n    , \"min_rows\" : 5\n    , \"seed\": 4241\n    , \"score_tree_interval\": 100,\"stopping_metric\" :\"MSE\",\"nfolds\":8,\"fold_assignment\":\"AUTO\",\"keep_cross_validation_predictions\" : True\n\n }\nfrom h2o.estimators.gbm import H2OGradientBoostingEstimator  # import gbm estimator\nmodel_gbm = H2OGradientBoostingEstimator(**param)\nmodel_gbm.train(x = train_list, y = 'result', training_frame = train_1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### GBM Model summary"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_gbm.summary","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Ensembling the XGBoost and GBM Model using GLM as meta algorithm"},{"metadata":{"trusted":true},"cell_type":"code","source":"from h2o.estimators import H2OStackedEnsembleEstimator\nstack = H2OStackedEnsembleEstimator(model_id=\"ensemble11\",\n                                       training_frame=train_1,\n                                       #validation_frame=test,\n                                       base_models=[model_xgb.model_id,model_gbm.model_id],metalearner_algorithm=\"glm\")\nstack.train(x=train_list, y=\"result\", training_frame=train_1)\n#stack.model_performance()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Summary of ensemble model"},{"metadata":{"trusted":true},"cell_type":"code","source":"stack.summary","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Not using the stacked ensemble model as it is performing relatively bad, so doing manual emsemble below"},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_1=h2o.H2OFrame(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred1=model_xgb.predict(test_1)\npred2=model_gbm.predict(test_1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df1=pred1.as_data_frame()\npred_df2=pred2.as_data_frame()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pred_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Ensemble of GBM with weight 0.9 and XGB weight 0.1"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = pred_df2['p1']*0.9+pred_df1['p1']*0.1\nsubmission_df\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### To be continued.... will add random forest and deep learning models. Stay Tuned. Please upvote if found useful"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}