{"cells":[{"metadata":{},"cell_type":"markdown","source":"# NCAAW 2020 - LGB w/ FE on three Datasets\n\nThis notebook shows LGB model training with feature engineering on three different datasets:\n- WRegularSeasonCompactResults\n- WRegularSeasonDetailedResults\n- WNCAATourneySeeds\n\nThe engineered features are appended to WNCAATourneyCompactResults and then LGB will be trained on it.\n\nOnly Season < 2015 is used for Stage1 training, so there is no leak on the test prediction."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns; sns.set()\n\nfrom sklearn.model_selection import GroupKFold, KFold\nfrom sklearn.metrics import log_loss\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For Stage 1, 2003 <= Season < 2015 will be used for the model training/validation and test preditions are calculated by that trained model. For Stage 2, Season >= 2003 will be used for  training."},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = '/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament'\n\nSTAGE_1 = True # This needs to be False when it's stage 2 ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## FE on RegularSeasonCompactResults\n\n### Calculating Win %"},{"metadata":{"trusted":true},"cell_type":"code","source":"WRSCResults = pd.read_csv(DATA_DIR + '/WDataFiles_Stage1/WRegularSeasonCompactResults.csv')\n\nA_w = WRSCResults[WRSCResults.WLoc == 'A']\\\n    .groupby(['Season','WTeamID'])['WTeamID'].count().to_frame()\\\n    .rename(columns={\"WTeamID\": \"win_A\"})\nN_w = WRSCResults[WRSCResults.WLoc == 'N']\\\n    .groupby(['Season','WTeamID'])['WTeamID'].count().to_frame()\\\n    .rename(columns={\"WTeamID\": \"win_N\"})\nH_w = WRSCResults[WRSCResults.WLoc == 'H']\\\n    .groupby(['Season','WTeamID'])['WTeamID'].count().to_frame()\\\n    .rename(columns={\"WTeamID\": \"win_H\"})\nwin = A_w.join(N_w, how='outer').join(H_w, how='outer').fillna(0)\n\nH_l = WRSCResults[WRSCResults.WLoc == 'A']\\\n    .groupby(['Season','LTeamID'])['LTeamID'].count().to_frame()\\\n    .rename(columns={\"LTeamID\": \"lost_H\"})\nN_l = WRSCResults[WRSCResults.WLoc == 'N']\\\n    .groupby(['Season','LTeamID'])['LTeamID'].count().to_frame()\\\n    .rename(columns={\"LTeamID\": \"lost_N\"})\nA_l = WRSCResults[WRSCResults.WLoc == 'H']\\\n    .groupby(['Season','LTeamID'])['LTeamID'].count().to_frame()\\\n    .rename(columns={\"LTeamID\": \"lost_A\"})\nlost = A_l.join(N_l, how='outer').join(H_l, how='outer').fillna(0)\n\nwin.index = win.index.rename(['Season', 'TeamID'])\nlost.index = lost.index.rename(['Season', 'TeamID'])\nwl = win.join(lost, how='outer').reset_index()\nwl['win_pct_A'] = wl['win_A'] / (wl['win_A'] + wl['lost_A'])\nwl['win_pct_N'] = wl['win_N'] / (wl['win_N'] + wl['lost_N'])\nwl['win_pct_H'] = wl['win_H'] / (wl['win_H'] + wl['lost_H'])\nwl['win_pct_All'] = (wl['win_A'] + wl['win_N'] + wl['win_H']) / \\\n    (wl['win_A'] + wl['win_N'] + wl['win_H'] + wl['lost_A']\\\n     + wl['lost_N'] + wl['lost_H'])\n\ndel A_w, N_w, H_w, H_l, N_l, A_l, win, lost","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Creating Score Features"},{"metadata":{"trusted":true},"cell_type":"code","source":"WRSCResults['relScore'] = WRSCResults.WScore - WRSCResults.LScore\n\nw_scr = WRSCResults.loc[:, ['Season', 'WTeamID', 'WScore', 'WLoc','relScore']]\nw_scr.columns = ['Season', 'TeamID','Score','Loc','relScore']\nl_scr = WRSCResults.loc[:, ['Season', 'LTeamID', 'LScore', 'WLoc','relScore']]\nl_scr['WLoc'] = l_scr.WLoc.apply(lambda x: 'H' if x == 'A' else 'A' \\\n                                 if x == 'H' else 'N')\nl_scr['relScore'] = -1 * l_scr.relScore \nl_scr.columns = ['Season', 'TeamID','Score','Loc','relScore']\nwl_scr = pd.concat([w_scr,l_scr])\n\nA_scr = wl_scr[wl_scr.Loc == 'A'].groupby(['Season','TeamID'])\\\n        ['Score','relScore'].mean()\\\n        .rename(columns={\"Score\": \"Score_A\", \"relScore\": \"relScore_A\"})\nN_scr = wl_scr[wl_scr.Loc == 'N'].groupby(['Season','TeamID'])\\\n        ['Score','relScore'].mean()\\\n        .rename(columns={\"Score\": \"Score_N\", \"relScore\": \"relScore_N\"})\nH_scr = wl_scr[wl_scr.Loc == 'H'].groupby(['Season','TeamID'])\\\n        ['Score','relScore'].mean()\\\n        .rename(columns={\"Score\": \"Score_H\", \"relScore\": \"relScore_H\"})\nAll_scr = wl_scr.groupby(['Season','TeamID'])['Score','relScore']\\\n    .mean().rename(columns={\"Score\": \"Score_All\", \"relScore\": \"relScore_All\"})\nscr = A_scr.join(N_scr, how='outer').join(H_scr, how='outer')\\\n    .join(All_scr, how='outer').fillna(0).reset_index()\n\ndel w_scr, l_scr, wl_scr, A_scr, H_scr, N_scr, All_scr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## FE on RegularSeasonDetailedResults"},{"metadata":{"trusted":true},"cell_type":"code","source":"WRSDetailedResults = pd.read_csv(DATA_DIR + '/WDataFiles_Stage1/WRegularSeasonDetailedResults.csv')\n\nw = WRSDetailedResults.loc[:, ['Season', 'WTeamID', 'WFGM','WFGA','WFGM3'\n                               ,'WFGA3','WFTM','WFTA','WOR','WDR','WAst',\n                               'WTO','WStl','WBlk','WPF']]\nw.columns = ['Season', 'TeamID', 'FGM','FGA','FGM3','FGA3','FTM','FTA','OR','DR',\n             'Ast','TO','Stl','Blk','PF']\nl = WRSDetailedResults.loc[:, ['Season', 'LTeamID', 'LFGM','LFGA','LFGM3',\n                               'LFGA3','LFTM','LFTA','LOR','LDR','LAst',\n                               'LTO','LStl','LBlk','LPF']]\nl.columns = ['Season', 'TeamID', 'FGM','FGA','FGM3','FGA3','FTM','FTA','OR','DR',\n             'Ast','TO','Stl','Blk','PF']\n\ndetail = pd.concat([w,l])\ndetail['goal_rate'] = detail.FGM / detail.FGA \ndetail['3p_goal_rate'] = detail.FGM3 / detail.FGA3  \ndetail['ft_goal_rate'] = detail.FTM  / detail.FTA  \n\ndt = detail.groupby(['Season','TeamID'])['FGM','FGA','FGM3','FGA3','FTM','FTA',\n                                         'OR','DR','Ast','TO','Stl','Blk','PF',\n                                          'goal_rate', '3p_goal_rate',\n                                         'ft_goal_rate']\\\n                                        .mean().fillna(0).reset_index()\n\ndel w, l, detail","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## FE on NCAATourneySeeds"},{"metadata":{"trusted":true},"cell_type":"code","source":"WNCAATourneySeeds = pd.read_csv(DATA_DIR + '/WDataFiles_Stage1/WNCAATourneySeeds.csv')\nWNCAATourneySeeds['seed_num'] =  WNCAATourneySeeds.Seed\\\n                                 .apply(lambda x: int(x[1:3]))\nseed = WNCAATourneySeeds.drop('Seed', axis=1)\n\ndel WNCAATourneySeeds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Duplicating each data with changing column names to be matched to 'WTeamID' and 'LTeamID' in Tourney dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"wl_1 = wl.loc[:,['Season','TeamID','win_pct_A','win_pct_N',\n                 'win_pct_H','win_pct_All']]\nwl_1.columns = [str(col) + '_1' if col not in ['Season','TeamID'] \\\n                else str(col) for col in wl_1.columns ]\n\nwl_2 = wl.loc[:,['Season','TeamID','win_pct_A','win_pct_N',\n                 'win_pct_H','win_pct_All']]\nwl_2.columns = [str(col) + '_2' if col not in ['Season','TeamID'] \\\n                else str(col) for col in wl_2.columns ]\n\nscr_1 = scr.copy()\nscr_1.columns = [str(col) + '_1' if col not in ['Season','TeamID'] \\\n                 else str(col) for col in scr_1.columns ]\n\nscr_2 = scr.copy()\nscr_2.columns = [str(col) + '_2' if col not in ['Season','TeamID'] \\\n                 else str(col) for col in scr_2.columns ]\n\ndt_1 = dt.copy()\ndt_1.columns = [str(col) + '_1' if col not in ['Season','TeamID'] \\\n                else str(col) for col in dt_1.columns ]\n\ndt_2 = dt.copy()\ndt_2.columns = [str(col) + '_2' if col not in ['Season','TeamID'] \\\n                else str(col) for col in dt_2.columns ]\n\nSeed_1 = seed.copy()\nSeed_1.columns = [str(col) + '_1' if col not in ['Season','TeamID'] \\\n                  else str(col) for col in Seed_1.columns ]\n\nSeed_2 = seed.copy()\nSeed_2.columns = [str(col) + '_2' if col not in ['Season','TeamID'] \\\n                  else str(col) for col in Seed_2.columns ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading NCAATourneyCompactResults\n\nThis dataset will be the base dataset for the model training"},{"metadata":{"trusted":true},"cell_type":"code","source":"TCResults = pd.read_csv(DATA_DIR + '/WDataFiles_Stage1/WNCAATourneyCompactResults.csv')\n\ntourney1 = TCResults.loc[:, ['Season','WTeamID','LTeamID']]\ntourney1.columns = ['Season','TeamID1','TeamID2']\ntourney1['result'] = 1\n\ntourney2 = TCResults.loc[:, ['Season','LTeamID','WTeamID']]\ntourney2.columns = ['Season','TeamID1','TeamID2']\ntourney2['result'] = 0\n\ntourney = pd.concat([tourney1, tourney2])\ndel tourney1, tourney2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Merging engineered features to Tourney dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge_data(df):\n\n    df = df.merge(wl_1, how='left', left_on=['Season','TeamID1'],\n                  right_on=['Season','TeamID'])\n    df = df.merge(wl_2, how='left', left_on=['Season','TeamID2'],\n                  right_on=['Season','TeamID'])\n    df = df.drop(['TeamID_x','TeamID_y'], axis=1)\n\n\n    df = df.merge(scr_1, how='left', left_on=['Season','TeamID1'],\n                  right_on=['Season','TeamID'])\n    df = df.merge(scr_2, how='left', left_on=['Season','TeamID2'],\n                  right_on=['Season','TeamID'])\n    df = df.drop(['TeamID_x','TeamID_y'], axis=1)\n\n    # df['win_pct_A_diff'] = df['win_pct_A_1'] - df['win_pct_A_2']\n    # df['win_pct_N_diff'] = df['win_pct_N_1'] - df['win_pct_N_2']\n    # df['win_pct_H_diff'] = df['win_pct_H_1'] - df['win_pct_H_2']\n    # df['win_pct_All_diff'] = df['win_pct_All_1'] - df['win_pct_All_2']\n\n    # df['Score_A_diff'] = df['Score_A_1'] - df['Score_A_2']\n    # df['Score_N_diff'] = df['Score_N_1'] - df['Score_N_2']\n    # df['Score_H_diff'] = df['Score_H_1'] - df['Score_H_2']\n    # df['Score_All_diff'] = df['Score_All_1'] - df['Score_All_2']\n\n    # df['relScore_A_diff'] = df['relScore_A_1'] - df['relScore_A_2']\n    # df['relScore_N_diff'] = df['relScore_N_1'] - df['relScore_N_2']\n    # df['relScore_H_diff'] = df['relScore_H_1'] - df['relScore_H_2']\n    # df['relScore_All_diff'] = df['relScore_All_1'] - df['relScore_All_2']\n\n    df = df.merge(dt_1, how='left', left_on=['Season','TeamID1'],\n                  right_on=['Season','TeamID'])\n    df = df.merge(dt_2, how='left', left_on=['Season','TeamID2'],\n                  right_on=['Season','TeamID'])\n    \n    df = df.drop(['TeamID_x','TeamID_y'], axis=1)\n    \n    df = df.merge(Seed_1, how='left', left_on=['Season','TeamID1'],\n                  right_on=['Season','TeamID'])\n    df = df.merge(Seed_2, how='left', left_on=['Season','TeamID2'],\n                  right_on=['Season','TeamID'])\n    df = df.drop(['TeamID_x','TeamID_y'], axis=1)\n\n    df['seed_num_diff'] = df['seed_num_1'] - df['seed_num_2']\n\n    df = df.fillna(-1)\n    \n    for col in df.columns:\n        if (df[col] == np.inf).any() or (df[col] == -np.inf).any():\n            df[col][(df[col] == np.inf) | (df[col] == -np.inf)] = -1\n    \n    return df\n\ntourney = merge_data(tourney)\ntourney = tourney.loc[tourney.Season >= 2003,:].reset_index(drop=True)\n\nif STAGE_1:\n    tourney = tourney.loc[tourney.Season < 2015, :]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading Submission Dataset\nDuplicating each ID with swapping TeamIDs. Predictions will be averaged by ID to get better performance."},{"metadata":{"trusted":true},"cell_type":"code","source":"if STAGE_1:\n    WSampleSubmission = pd.read_csv(DATA_DIR + '/WSampleSubmissionStage1_2020.csv')\nelse:\n    WSampleSubmission = pd.read_csv(DATA_DIR + None) # put stage 2 submission file link here\n\ntest1 = WSampleSubmission.copy()\ntest1['Season'] = test1.ID.apply(lambda x: int(x[0:4]))\ntest1['TeamID1'] = test1.ID.apply(lambda x: int(x[5:9]))\ntest1['TeamID2'] = test1.ID.apply(lambda x: int(x[10:14]))\n\ntest2 = WSampleSubmission.copy()\ntest2['Season'] = test2.ID.apply(lambda x: int(x[0:4]))\ntest2['TeamID1'] = test2.ID.apply(lambda x: int(x[10:14]))\ntest2['TeamID2'] = test2.ID.apply(lambda x: int(x[5:9]))\n\ntest = pd.concat([test1,test2]).drop(['Pred'], axis=1)\ntest = merge_data(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"X = tourney.drop(['Season','TeamID1','TeamID2','result'], axis=1)\ny = tourney[\"result\"]\ns = tourney[\"Season\"]\n\nX_test = test.drop(['ID', 'Season','TeamID1','TeamID2'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_training(X, y, cv, groups, params, metric, early_stopping=10, \\\n    plt_iter=True, X_test=[], cat_features=[]):\n\n    feature_importance = pd.DataFrame()\n    val_scores=[]\n    train_evals=[]\n    valid_evals=[]\n\n    if len(X_test) > 0:\n        test_pred=np.zeros(len(X_test))\n\n    for idx, (train_index, val_index) in enumerate(cv.split(X, y, groups)):\n\n        print(\"###### fold %d ######\" % (idx+1))\n        X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n        y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n\n        model = lgb.LGBMClassifier(**params)\n\n        model.fit(X_train, y_train,\n                  eval_set=[(X_train, y_train), (X_val, y_val)],\n                  early_stopping_rounds=early_stopping,\n                  verbose=20\n                  #categorical_feature=list(cate_ft_lst),\n                  )\n\n        val_scores.append(model.best_score_['valid_1'][metric])\n        train_evals.append(model.evals_result_['training'][metric])\n        valid_evals.append(model.evals_result_['valid_1'][metric])\n\n        if len(X_test) > 0:\n            test_pred = test_pred + model.predict_proba(X_test)[:,1]\n\n        fold_importance = pd.DataFrame()\n        fold_importance[\"feature\"] = X_train.columns\n        fold_importance[\"importance\"] = model.feature_importances_\n        fold_importance[\"fold\"] = idx+1\n        feature_importance = pd.concat([feature_importance, fold_importance]\n                                       , axis=0)\n\n    if plt_iter:\n        \n        fig, axs = plt.subplots(2, 2, figsize=(9,6))\n        \n        for i, ax in enumerate(axs.flatten()):\n            ax.plot(train_evals[i], label='training')\n            ax.plot(valid_evals[i], label='validation')\n            ax.set(xlabel='interations', ylabel=f'{metric}')\n            ax.set_title(f'fold {i+1}', fontsize=12)\n            ax.legend(loc='upper right', prop={'size': 9})\n        fig.tight_layout()\n        plt.show()\n    \n    print('### CV scores by fold ###')\n    for i in range(cv.get_n_splits(X)):\n        print(f'fold {i+1}: {val_scores[i]:.4f}')\n    print('CV mean score: {0:.4f}, std: {1:.4f}.'\\\n          .format(np.mean(val_scores), np.std(val_scores)))\n    \n    feature_importance = feature_importance[[\"feature\", \"importance\"]]\\\n                         .groupby(\"feature\").mean().sort_values(\n                         by=\"importance\", ascending=False)\n    feature_importance.reset_index(inplace=True)\n\n    if len(X_test) > 0:\n        test_pred = test_pred / cv.get_n_splits(X)\n        return feature_importance, test_pred\n    else:\n        return feature_importance","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_params = {'objective': 'binary',\n              'metric': 'binary_logloss',\n              'boosting': 'gbdt',\n              'num_leaves': 32,\n              'feature_fraction': 0.6,\n              'bagging_fraction': 0.6,\n              'bagging_freq': 5,\n              'learning_rate': 0.05\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using GroupKFold by Season"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngroup_kfold = GroupKFold(n_splits=4)\n\nfeature_importance, test_pred = \\\n    model_training(X, y, group_kfold, s, lgb_params, \n    'binary_logloss', plt_iter=True, X_test=X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 10));\nsns.barplot(x=\"importance\", y=\"feature\", data=feature_importance[:30])\nplt.title('Feature Importnace')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating Submission File\n\nThe second half of the test prediction need to be (1 - pred) as the team order was swapped. The predictions are averaged by ID after that."},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = test_pred.shape[0] //2\ntest_pred[idx:] = 1 - test_pred[idx:]\n\npred = pd.concat([test.ID, pd.Series(test_pred)], axis=1).groupby('ID')[0]\\\n        .mean().reset_index().rename(columns={0:'Pred'})\nsub = WSampleSubmission.drop(['Pred'],axis=1).merge(pred, on='ID')\nsub.to_csv('submission.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Calculating Test Score Locally\nFor some reasons, the score here is slightly different from LB"},{"metadata":{"trusted":true},"cell_type":"code","source":"if STAGE_1:\n    rslt = pd.DataFrame()\n    TCResults_s = TCResults.loc[TCResults.Season >= 2015,:]\n    rslt['season'] = TCResults_s.Season\n    rslt['team1'] = TCResults_s.apply(lambda x: x.WTeamID \\\n                                      if x.WTeamID < x.LTeamID else x.LTeamID\n                                      , axis=1)\n    rslt['team2'] = TCResults_s.apply(lambda x: x.WTeamID \\\n                                      if x.WTeamID > x.LTeamID else x.LTeamID\n                                      , axis=1)\n    rslt['wl'] = TCResults_s.apply(lambda x: 1 if x.WTeamID < x.LTeamID else 0\n                                   , axis=1)\n    rslt['ID'] = rslt.apply(lambda x: str(x.season) + '_' + str(x.team1) \\\n                            + '_' + str(x.team2), axis=1)\n    sub2 = sub.merge(rslt.loc[:,['ID','wl']], how='inner', on='ID')\n\n    preds = []\n    for i in sub2.Pred:\n        preds.append([1-i, i])\n\n    print('Test logloss is {:.5f}'.format(log_loss(sub2.wl.values, preds)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}