{"cells":[{"metadata":{},"cell_type":"markdown","source":"Referenced kernel\n\n[MM NCAAM No leaks: LGB, XGB, LogReg](https://www.kaggle.com/vbmokin/mm-ncaam-no-leaks-lgb-xgb-logreg)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Libraries Import"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV\nfrom sklearn import preprocessing\nimport lightgbm as lgb\nimport optuna\nimport glob","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/'\nFiles = 'WDataFiles_Stage1/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TourneyCompactResults = pd.read_csv(path + Files + 'WNCAATourneyCompactResults.csv')\nGameCities = pd.read_csv(path+Files+'WGameCities.csv')\nSeasons = pd.read_csv(path+Files+'WSeasons.csv')\nTourneySeeds = pd.read_csv(path+Files+'WNCAATourneySeeds.csv')\nRegularSeasonCompactResults = pd.read_csv(path+Files+'WRegularSeasonCompactResults.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test= pd.read_csv(path +'WSampleSubmissionStage1_2020.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Overview"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(TourneyCompactResults.shape)\nTourneyCompactResults.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(GameCities.shape)\nGameCities.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(Seasons.shape)\nSeasons.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TourneySeeds['Seed'] = TourneySeeds['Seed'].apply(lambda x: int(x[1:3]))\nprint(TourneySeeds.shape)\nTourneySeeds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(RegularSeasonCompactResults.shape)\nRegularSeasonCompactResults.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = TourneyCompactResults\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.replace({'H':0,'A':1,'N':2})\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = preprocessing.LabelEncoder()\nfor column in ['CRType']:\n    le.fit(GameCities[column])\n    GameCities[column] = le.transform(GameCities[column])\nGameCities.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = train.merge(GameCities,how='left',on=['Season','WTeamID','LTeamID'],indicator = False)\n#train.drop('DayNum_y',inplace=True,axis=1)\n#train = train.rename(columns={'DayNum_x': 'DayNum'})\n#train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = preprocessing.LabelEncoder()\nfor column in ['RegionW','RegionX','RegionY','RegionZ']:\n    le.fit(Seasons[column])\n    Seasons[column] = le.transform(Seasons[column])\n\nfor i in range(0,23):\n    print(Seasons['DayZero'][i].split('/'))\n    Seasons['ZeroMonth'] = Seasons['DayZero'][i].split('/')[0]\n    Seasons['ZeroDay'] = Seasons['DayZero'][i].split('/')[1]\n    Seasons['ZeroYear'] = Seasons['DayZero'][i].split('/')[2]\n\nSeasons = Seasons.drop('DayZero',axis=1)\nSeasons['ZeroMonth'] = Seasons['ZeroMonth'].astype(int)\nSeasons['ZeroDay'] = Seasons['ZeroMonth'].astype(int)\nSeasons['ZeroYear'] = Seasons['ZeroMonth'].astype(int)\nSeasons.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(Seasons, how='left',on=['Season'])\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(TourneySeeds, how='left', left_on=['Season', 'WTeamID'], right_on=['Season','TeamID'])\ntrain = train.drop('TeamID',axis=1)\ntrain = train.rename(columns={'Seed': 'WSeed'})\n\ntrain = train.merge(TourneySeeds, how='left', left_on=['Season', 'LTeamID'], right_on=['Season','TeamID'])\ntrain = train.drop('TeamID',axis=1)\ntrain = train.rename(columns={'Seed': 'LSeed'})\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# format ID\ntest = test.drop(['Pred'], axis=1)\ntest['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))\ntest['WTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[1]))\ntest['LTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[2]))\n\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.merge(TourneyCompactResults,how='left',on=['Season','WTeamID','LTeamID'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest = test.replace({'H':0,'A':1,'N':2})\n\n#test = test.merge(GameCities,how='left',on=['Season','WTeamID','LTeamID'])\n#test = test.rename(columns={'DayNum_x': 'DayNum'})\n#print(test.shape)\n\ntest = test.merge(Seasons, how='left',on=['Season'])\n\ntest = test.merge(TourneySeeds, how='left', left_on=['Season', 'WTeamID'], right_on=['Season','TeamID'])\ntest = test.drop('TeamID',axis=1)\ntest = test.rename(columns={'Seed': 'WSeed'})\n\n\ntest = test.merge(TourneySeeds, how='left', left_on=['Season', 'LTeamID'], right_on=['Season','TeamID'])\ntest = test.drop('TeamID',axis=1)\ntest = test.rename(columns={'Seed': 'LSeed'})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.merge(test,how='left',on=['ID','Season','WTeamID','LTeamID'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist()]\nprint(not_exist_in_test)\ntrain = train.drop(not_exist_in_test, axis=1)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#RegularSeasonCompactResults\n# split winners and losers\nteam_win_score = RegularSeasonCompactResults.groupby(['Season', 'WTeamID']).agg({'WScore':['sum', 'count', 'var']}).reset_index()\nteam_win_score.columns = [' '.join(col).strip() for col in team_win_score.columns.values]\nteam_loss_score = RegularSeasonCompactResults.groupby(['Season', 'LTeamID']).agg({'LScore':['sum', 'count', 'var']}).reset_index()\nteam_loss_score.columns = [' '.join(col).strip() for col in team_loss_score.columns.values]\n\nprint(team_win_score.shape)\nteam_win_score.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with train \ntrain = pd.merge(train, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])\ntrain = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])\ntrain = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])\ntrain = pd.merge(train, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])\ntrain.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge with test \ntest = pd.merge(test, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])\ntest = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])\ntest = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])\ntest = pd.merge(test, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])\ntest.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)\ntest.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(df):\n    df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']\n    df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']\n    df['x_count'] = df['WScore count_x'] + df['LScore count_y']\n    df['y_count'] = df['WScore count_y'] + df['WScore count_x']\n    df['x_var'] = df['WScore var_x'] + df['LScore count_y']\n    df['y_var'] = df['WScore var_y'] + df['WScore var_x']\n    return df\ntrain = preprocess(train)\ntest = preprocess(test)\ntest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make winner and loser train\ntrain_win = train.copy()\ntrain_los = train.copy()\ntrain_win = train_win[['WSeed', 'LSeed',\n                 'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntrain_los = train_los[['LSeed', 'WSeed', \n                 'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntrain_win.columns = ['Seed_1', 'Seed_2',\n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']\ntrain_los.columns = ['Seed_1', 'Seed_2', \n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']\n\n# same processing for test\ntest = test[['ID', 'WSeed', 'LSeed', \n                 'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]\ntest.columns = ['ID', 'Seed_1', 'Seed_2', \n                  'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature enginnering\ndef feature_engineering(df):\n    df['Seed_diff'] = df['Seed_1'] - df['Seed_2']\n    df['Score_diff'] = df['Score_1'] - df['Score_2']\n    df['Count_diff'] = df['Count_1'] - df['Count_2']\n    df['Var_diff'] = df['Var_1'] - df['Var_2']\n    df['Mean_score1'] = df['Score_1'] / df['Count_1']\n    df['Mean_score2'] = df['Score_2'] / df['Count_2']\n    df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']\n    df['FanoFactor_1'] = df['Var_1'] / df['Mean_score1']\n    df['FanoFactor_2'] = df['Var_2'] / df['Mean_score2']\n    return df\ntrain_win = feature_engineering(train_win)\ntrain_los = feature_engineering(train_los)\ntest = feature_engineering(test)\ntest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_win[\"result\"] = 1\nprint(train_win.shape)\ntrain_win.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_los[\"result\"] = 0\nprint(train_los.shape)\ntrain_los.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.concat((train_win, train_los)).reset_index(drop=True)\nprint(data.shape)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.drop(['ID'],axis=1)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nimport matplotlib.pyplot as plt\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import KFold\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom xgboost import XGBClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train=data['result']\nX_train=data.drop(columns='result')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params_lgb = {'num_leaves': 400,\n              'min_child_weight': 0.034,\n              'feature_fraction': 0.379,\n              'bagging_fraction': 0.418,\n              'min_data_in_leaf': 106,\n              'objective': 'binary',\n              'max_depth': 50,\n              'learning_rate': 0.0068,\n              \"boosting_type\": \"gbdt\",\n              \"bagging_seed\": 11,\n              \"metric\": 'logloss',\n              \"verbosity\": -1,\n              'reg_alpha': 0.3899,\n              'reg_lambda': 0.648,\n              'random_state': 47,\n              }\n\nparams_xgb = {'colsample_bytree': 0.8,                 \n              'learning_rate': 0.0004,\n              'max_depth': 31,\n              'subsample': 1,\n              'objective':'binary:logistic',\n              'eval_metric':'logloss',\n              'min_child_weight':3,\n              'gamma':0.25,\n              'n_estimators':5000\n              }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLDS = 10\nfolds = KFold(n_splits=NFOLDS)\n\ncolumns = X_train.columns\nsplits = folds.split(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds_lgb = np.zeros(test.shape[0])\ny_oof_lgb = np.zeros(X_train.shape[0])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\nprint(X_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold_n, (train_index, valid_index) in enumerate(splits):\n    print('Fold:',fold_n+1)\n    X_train1, X_valid1 = X_train[columns].iloc[train_index], X_train[columns].iloc[valid_index]\n    y_train1, y_valid1 = y_train.iloc[train_index], y_train.iloc[valid_index]\n    \n    dtrain = lgb.Dataset(X_train1, label=y_train1)\n    dvalid = lgb.Dataset(X_valid1, label=y_valid1)\n\n    clf = lgb.train(params_lgb, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)\n    \n    y_pred_valid = clf.predict(X_valid1)\n    y_oof_lgb[valid_index] = y_pred_valid\n    \n    y_preds_lgb += clf.predict(test) / NFOLDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds_lgb.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv(path + 'WSampleSubmissionStage1_2020.csv')\n#submission_df['Pred'] = 0.94*y_preds_lgb + 0.06*y_preds_xgb\nsubmission_df['Pred'] = y_preds_lgb\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test= pd.read_csv(path +'WSampleSubmissionStage1_2020.csv')\ntest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ToDo\n+ OverFitting\nhttps://www.kaggle.com/c/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/discussion/131028\n+ use Seeds"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}