{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Overview\n\nThis notebooks\ncontains \n* season score aggregation\n* one hot encoding for location\n* simple prediction with LightGBM, Xgboost, Neural Network\n* lag features (we can use data of past seasons) \n\nwill contain\n* enhancement of location for \"Neutral\"\n* model selection and hypermarameter chuning"},{"metadata":{},"cell_type":"markdown","source":"## Import Library & Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"results = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneyCompactResults.csv')\nseeds = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneySeeds.csv')\nsubmission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seeds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# it seems home team has high probability to win\nresults.WLoc.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#convert results which have result columns\ndef convert_results(results):\n    results['result'] = 1 # win\n    Lresults = results.copy()\n    Lresults['result'] = 0 # lose\n    Lresults['WTeamID'] = results['LTeamID']\n    Lresults['LTeamID'] = results['WTeamID']\n    Lresults['WScore'] = results['LScore']\n    Lresults['LScore'] = results['WScore']\n    Lresults['WLoc'].replace({'H': 'A'}, inplace=True)\n    results = pd.concat([results, Lresults]).reset_index(drop=True)\n    \n    results.rename(columns={'WTeamID': 'TeamID1', 'LTeamID': 'TeamID2'}, inplace=True)\n    results.rename(columns={'WScore': 'Score1', 'LScore': 'Score2'}, inplace=True)\n    results.rename(columns={'WLoc': 'Loc'}, inplace=True)\n\n    return results\n\ndata = convert_results(results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_seed(x):\n    return int(x[1:])\n\n#merge seed data\ndef setup_seed(results, seeds):\n    data = pd.merge(results, seeds, left_on=['Season', 'TeamID1'], right_on=['Season', 'TeamID'])\n    data.rename(columns={'Seed': \"Seed1\"}, inplace=True)\n    data.drop('TeamID', axis=1, inplace=True)\n\n    data = pd.merge(data, seeds, left_on=['Season', 'TeamID2'], right_on=['Season', 'TeamID'])\n    data.rename(columns={'Seed': \"Seed2\"}, inplace=True)\n    data.drop('TeamID', axis=1, inplace=True)\n    data['Seed1'] = data['Seed1'].map(get_seed)\n    data['Seed2'] = data['Seed2'].map(get_seed)\n    \n    data['seed_diff'] = data['Seed1'] - data['Seed2']\n    \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = setup_seed(data, seeds)\ndata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# aggregate seasonal socre\n# off_score is the score which a team got in the season\n# def_score is the score which a team lost in the season\n\nseason_score = data.groupby(['Season', 'TeamID1']).mean().reset_index()[['Season', 'TeamID1', 'Score1', 'Score2']]\nseason_score.rename(columns={'TeamID1': 'TeamID', 'Score1': 'off_score', 'Score2': 'def_score'}, inplace=True)\nseason_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lag_score(scores):\n    scores_1y = scores.copy()\n    scores_1y['Season'] += 1\n    scores_1y.rename(columns={'off_score': 'off_score_1y', 'def_score': 'def_score_1y'}, inplace=True)\n    scores = scores.merge(scores_1y, on=['Season', 'TeamID'], how='left')\n\n    return scores\nseason_score_lag = get_lag_score(season_score)\nseason_score_lag","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare data for training or testing\nSince given data for prediction is only season and teamIDs, we have to aggregate data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_location(data):\n    tmp = pd.get_dummies(data['Loc'], drop_first=True, prefix='location')\n    data = pd.concat([data, tmp], axis=1)\n    return data\n\n#convert Location to one hot vectors\ndata = convert_location(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"delete_columns = [\n    'DayNum',\n    'TeamID1',\n    'Score1',\n    'TeamID2',\n    'Score2',\n    'Loc',\n    'NumOT',\n    'result'\n]\n\ndef gen_datasets(data, season_score):\n    #merge season scores\n    data = pd.merge(data, season_score, left_on=['Season', 'TeamID1'], right_on=['Season', 'TeamID'])\n    data.rename(columns={'off_score': 'off_score1', 'def_score': 'def_score1'}, inplace=True)\n    data.rename(columns={'off_score_1y': 'off_score1_1y', 'def_score_1y': 'def_score1_1y'}, inplace=True)\n\n    data.drop('TeamID', axis=1, inplace=True)\n    data = pd.merge(data, season_score, left_on=['Season', 'TeamID2'], right_on=['Season', 'TeamID'])\n    data.rename(columns={'off_score': 'off_score2', 'def_score': 'def_score2'}, inplace=True)\n    data.rename(columns={'off_score_1y': 'off_score2_1y', 'def_score_1y': 'def_score2_1y'}, inplace=True)\n\n    data.drop('TeamID', axis=1, inplace=True)\n    \n    #compare seasonal scores\n    data['score_diff1'] = data['off_score1'] - data['def_score2']\n    data['score_diff2'] = data['off_score2'] - data['def_score1']\n\n    y = data['result']\n    X = data.drop(delete_columns, axis=1)\n   \n    return X, y\n\ntrain_x, train_y = gen_datasets(data, season_score_lag)\ntrain_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#only for Stage1\ndef prepare_test(df, data):\n    df['Season'] = df['ID'].map(lambda x:int(x.split('_')[0]))\n    df['TeamID1'] = df['ID'].map(lambda x:int(x.split('_')[1]))\n    df['TeamID2'] = df['ID'].map(lambda x:int(x.split('_')[2]))\n    #df.drop('ID', axis=1, inplace=True)\n    \n    tmp = data.drop_duplicates(['Season', 'TeamID1', 'Seed1'])\n    df = pd.merge(df, tmp[['Season', 'TeamID1', 'Seed1']],  on=['Season', 'TeamID1'], how='inner')\n    tmp = data.drop_duplicates(['Season', 'TeamID2', 'Seed2'])\n    df = pd.merge(df, tmp[['Season', 'TeamID2', 'Seed2']],  on=['Season', 'TeamID2'], how='inner') \n    df['seed_diff'] = df['Seed1'] - df['Seed2']\n    \n    df['Loc'] = pd.merge(df, data[['Season', 'TeamID1', 'TeamID2', 'Loc']],  on=['Season', 'TeamID1', 'TeamID2'], how='inner')['Loc']\n    df['Loc'].fillna('N', inplace=True)\n    df = convert_location(df)\n\n    #insert dummy columns\n    df['result'] = 9999\n    df['DayNum'] = 9999\n    df['Score1'] = 9999\n    df['Score2'] = 9999\n    df['NumOT'] = 9999\n\n    test_x, _ = gen_datasets(df, season_score_lag)\n    \n    return test_x\ntest_x = prepare_test(submission_df, data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x.columns, test_x.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build [LightGBM](https://lightgbm.readthedocs.io/en/latest/) Model\n\nnow, let's build a simple model and predict result\n\nNote: To make codes simple, we accept leak at the moment"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgbm\nparams_lgb = {'num_leaves': 127,\n          'min_data_in_leaf': 10,\n          'objective': 'binary',\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting_type\": \"gbdt\",\n          \"bagging_seed\": 11,\n          \"metric\": 'logloss',\n          \"verbosity\": 0\n          }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_lgb = lgbm.Dataset(train_x, train_y)\nclf_lgb = lgbm.train(params_lgb, train_lgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y_lgb = pd.Series(clf_lgb.predict(test_x.drop(['ID','Pred'], axis=1)), name='Pred')\npred_y_lgb = pd.concat([test_x['ID'], pred_y_lgb], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y_lgb.Pred.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(clf_lgb.feature_importance(), index=train_x.columns).sort_values().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build XGBModel"},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb\nparams_xgb = {'max_depth': 50,\n              'objective': 'binary:logistic',\n              'eta'      : 0.3,\n              'subsample': 0.8,\n              'lambda '  : 4,\n              'eval_metric': 'logloss',\n              'n_estimators': 1000,\n              'colsample_bytree ': 0.9,\n              'colsample_bylevel': 1\n              }\ntrain_xgb = xgb.DMatrix(train_x, train_y)\nclf_xgb = xgb.train(params_xgb, train_xgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y_xgb = pd.Series(clf_xgb.predict(xgb.DMatrix(test_x.drop(['ID','Pred'], axis=1))), name='Pred')\npred_y_xgb = pd.concat([test_x['ID'], pred_y_xgb], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y_xgb.Pred.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(clf_lgb.feature_importance(), index=train_x.columns).sort_values().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build NN model\nbuild NN model with keras w/ tensorflow backend"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Input, Dense, Dropout, Activation, BatchNormalization\nfrom sklearn.model_selection import train_test_split\ndef gen_NN_model():\n    model = Sequential()\n    model.add(Dense(128, input_shape=(16, )))\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    model.add(Dropout(0.2))\n    model.add(Dense(256))\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    model.add(Dropout(0.2))\n    model.add(Dense(512, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    model.add(Dense(1))\n    model.add(Activation('sigmoid'))\n    #model.summary()\n    model.compile(loss='binary_crossentropy', optimizer='adam')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NN_model = gen_NN_model()\nNN_model.fit(train_x.fillna(-9999), train_y, batch_size=100,\n            epochs=20, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y_nn = pd.Series(NN_model.predict(test_x.drop(['ID','Pred'], axis=1).fillna(-9999)).reshape(len(test_x)), name='Pred')\npred_y_nn = pd.concat([test_x['ID'], pred_y_nn], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y_nn.Pred.hist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Comparison"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import log_loss\ndef mixup_prediction(true_y, pred_lgb, pred_xgb, pred_nn):\n    min_loss = 10\n    min_w1 = 10\n    min_w2 = 10\n    for w1 in np.linspace(0, 1):\n        for w2 in np.linspace(0, 1-w1):\n            pred_mix = pred_lgb*w1 + pred_xgb*w2 + pred_nn*(1-w1-w2)\n            ans = log_loss(true_y, pred_mix)\n            if ans<min_loss: \n                min_w1 = w1\n                min_w2 = w2\n                min_loss = ans\n    print('log_loss: {}, lgb_weight: {}, xgb_weight:{}, NN_weight:{}'.format(min_loss, min_w1, min_w2, 1-min_w1-min_w2))\n    return min_w1, min_w2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nkf = KFold(n_splits=5)\nloss_lgb = []\nloss_xgb = []\nloss_nn = []\nweights = []\n\nfor train_index, val_index in kf.split(train_x):\n    tr_x = train_x.iloc[train_index]\n    va_x = train_x.iloc[val_index]\n    tr_y = train_y.iloc[train_index]\n    va_y = train_y.iloc[val_index]\n    \n    #light gbm\n    train_lgb = lgbm.Dataset(tr_x, tr_y)\n    lgb_clf = lgbm.train(params_lgb, train_lgb)\n    pred_lgb = lgb_clf.predict(va_x)\n    loss_lgb.append(log_loss(va_y, pred_lgb))\n    \n    #xgboost\n    train_xgb = xgb.DMatrix(tr_x, tr_y)\n    clf_xgb = xgb.train(params_xgb, train_xgb)\n    pred_xgb = pd.Series(clf_xgb.predict(xgb.DMatrix(va_x)), name='Pred')\n    loss_xgb.append(log_loss(va_y, pred_xgb))\n    \n    #neural network\n    NN_model = gen_NN_model()\n    NN_model.fit(tr_x.fillna(-9999), tr_y, batch_size=100, epochs=20, verbose=0)\n    pred_nn = NN_model.predict(va_x.fillna(-9999)).reshape(len(va_x))\n    loss_nn.append(log_loss(va_y, pred_nn))\n    \n    weights.append(mixup_prediction(va_y, pred_lgb, pred_xgb, pred_nn))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.mean(loss_lgb), np.mean(loss_xgb), np.mean(loss_nn)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"xgboost seems best model...[](http://)"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x['Pred'] = clf_xgb.predict(xgb.DMatrix(test_x.drop(['ID','Pred'], axis=1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x['Pred'] = test_x['Pred'].clip(0, 1)\ntest_x['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x[['ID', 'Pred']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}