{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os\nimport warnings\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.model_selection import StratifiedKFold, RepeatedStratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler\nfrom sklearn.metrics import log_loss\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Preprocessing"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneyCompactResults.csv')\nresult = result.drop(columns=['WLoc', 'NumOT', 'DayNum'])\nresult.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seeds = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneySeeds.csv')\nseeds.Seed = seeds.Seed.map(lambda string : int(string[1:3]))\nseeds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Wseeds = seeds.rename(columns={'TeamID':'WTeamID', 'Seed':'WSeed'})\nLseeds = seeds.rename(columns={'TeamID':'LTeamID', 'Seed':'LSeed'})\n\ndata = pd.merge(left=result, right=Wseeds, how='left', on=['Season', 'WTeamID'])\ndata = pd.merge(left=data, right=Lseeds, on=['Season', 'LTeamID'])\n\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WRegularSeasonCompactResults.csv')\nscores.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Lscores = scores[['Season', 'WTeamID', 'WScore']].rename(columns={'WTeamID':'TeamID', 'WScore':'Score'})\nWscores = scores[['Season', 'LTeamID', 'LScore']].rename(columns={'LTeamID':'TeamID', 'LScore':'Score'})\n\nresult_scores = pd.concat([Wscores, Lscores])\nresult_scores.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"season_score = result_scores.groupby(['Season', 'TeamID'])['Score'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.merge(data, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ndata = data.rename(columns={'Score':'WScoreT'})\ndata = pd.merge(data, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ndata = data.rename(columns={'Score':'LScoreT'})\ndata = data.drop(columns=['WScore', 'LScore'])\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Wdata = data.drop(columns=['Season', \n                           #'WTeamID', \n                           #'LTeamID'\n                           ])\nWdata.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', \n                      'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2',\n                      'WTeamID':'TeamID_1', 'LTeamID': 'TeamID_2'}, inplace=True)\nWdata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ldata = data[['LTeamID', 'WTeamID', 'LSeed', 'WSeed', 'LScoreT', 'WScoreT']]\nLdata.rename(columns={'LTeamID':'TeamID_1', 'WTeamID':'TeamID_2', \n                      'LSeed':'Seed1', 'WSeed':'Seed2', \n                      'LScoreT':'ScoreT1', 'WScoreT':'ScoreT2',}, inplace=True)\nLdata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Wdata['Seed_diff'] = Wdata['Seed1'] - Wdata['Seed2']\nWdata['ScoreT_diff'] = Wdata['ScoreT1'] - Wdata['ScoreT2']\nLdata['Seed_diff'] = Ldata['Seed1'] - Ldata['Seed2']\nLdata['ScoreT_diff'] = Ldata['ScoreT1'] - Ldata['ScoreT2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Wdata['result'] = 1\nLdata['result'] = 0\ntrain = pd.concat((Wdata, Ldata)).reset_index(drop=True)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract year and ID number out of string\ntest = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\ntest['Season'] = test.ID.map(lambda string : int(string.split('_')[0]))\ntest['TeamID_1'] = test.ID.map(lambda string : int(string.split('_')[1]))\ntest['TeamID_2'] = test.ID.map(lambda string : int(string.split('_')[2]))\ntest = test.drop(columns=['ID'])\ntest.head()\n\n# Convert test data to the train set's format\ntest = pd.merge(test, seeds, left_on=['Season', 'TeamID_1'], right_on=['Season', 'TeamID'], how='left')\ntest.rename(columns={'Seed':'Seed1'}, inplace=True)\ntest = test.drop('TeamID', axis=1)\ntest = pd.merge(test, seeds, left_on=['Season', 'TeamID_2'], right_on=['Season', 'TeamID'], how='left')\ntest.rename(columns={'Seed':'Seed2'}, inplace=True)\ntest = test.drop('TeamID', axis=1)\ntest = pd.merge(test, season_score, left_on=['Season', 'TeamID_1'], right_on=['Season', 'TeamID'], how='left')\ntest.rename(columns={'Score':'ScoreT1'}, inplace=True)\ntest = pd.merge(test, season_score, left_on=['Season', 'TeamID_2'], right_on=['Season', 'TeamID'], how='left')\ntest.rename(columns={'Score':'ScoreT2'}, inplace=True)\ntest['Seed_diff'] = test['Seed1'] - test['Seed2']\ntest['ScoreT_diff'] = test['ScoreT1'] - test['ScoreT2']\n#test = test.drop(columns=['Pred', 'Season', 'TeamID_1', 'TeamID_2'])\ntest = test.drop(columns=['Pred', 'Season'])\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.decomposition import PCA\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.TeamID_1.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_EMBINT = max(train.TeamID_1.unique())+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X, y = train.drop('result', axis=1).values, train['result'].values\nX_test = test.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X[:, 2:] = scaler.fit_transform(X[:, 2:])\nX_test[:, 2:] = scaler.transform(X_test[:, 2:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca = PCA(n_components=2)\nx_pca = pca.fit_transform(X[:, 2:])\n\nplt.plot(x_pca[train['result'].values==1, 0], x_pca[train['result'].values==1, 1], '.g', alpha=0.25)\nplt.plot(x_pca[train['result'].values==0, 0], x_pca[train['result'].values==0, 1], '.r', alpha=0.25)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Keras Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorflow-addons","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mish(x):\n    return x * tf.keras.backend.softplus(tf.keras.backend.tanh(x))\n\ntf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/jarnel/clipping-spline-experiment-on-test-predictions\n\nfrom scipy.interpolate import UnivariateSpline\n\ndef spline_model(labels, preds):\n    comb = pd.DataFrame({'labels':labels, 'preds':preds})\n    comb = comb.sort_values(by='preds').reset_index(drop=True)\n    spline_model = UnivariateSpline(comb['preds'].values, comb['labels'].values)\n    adjusted = spline_model(preds)\n    return spline_model, log_loss(labels, adjusted)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef focal_loss(gamma=1.5, alpha=.5):\n\tdef focal_loss_fixed(y_true, y_pred):\n\t\tpt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n\t\tpt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\t\treturn -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\treturn focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    feature_inp = tf.keras.layers.Input((6,), name='FeatureInput')\n    id1_inp = tf.keras.layers.Input((1,), name='ID1Input')\n    id2_inp = tf.keras.layers.Input((1,), name='ID2Input')\n    \n    emb = tf.keras.layers.Embedding(MAX_EMBINT, 1, input_length=1)\n    \n    e1 = tf.keras.layers.Flatten()(emb(id1_inp))\n    e2 = tf.keras.layers.Flatten()(emb(id2_inp))\n    \n    e1 = tf.keras.layers.Dropout(0.1)(e1)\n    e2 = tf.keras.layers.Dropout(0.1)(e2)\n    \n    x = tf.keras.layers.Dense(16)(feature_inp)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation(mish)(x)\n    x = tf.keras.layers.Dropout(0.25)(x)\n    \n    e = tf.keras.layers.Concatenate()([e1, e2])\n    e = tf.keras.layers.Dense(16)(e)\n    e = tf.keras.layers.BatchNormalization()(e)\n    e = tf.keras.layers.Activation(mish)(e)\n    e = tf.keras.layers.Dropout(0.25)(e)\n    \n    x = tf.keras.layers.Concatenate()([x, e])\n    \n    x = tf.keras.layers.Dense(16)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation(mish)(x)\n    x = tf.keras.layers.Dropout(0.25)(x)\n    \n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    \n    model = tf.keras.Model([feature_inp, id1_inp, id2_inp], x)\n    model.compile(optimizer=tfa.optimizers.RectifiedAdam(lr=1e-3),\n                  loss=focal_loss(), metrics=['binary_crossentropy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=10)\nlosses = []\npredicts = []\nfor i, (train_ind,valid_ind) in enumerate(cv.split(X, y)):\n    tf.keras.backend.clear_session()\n    \n    X_train, X_valid = X[train_ind], X[valid_ind]\n    y_train, y_valid = y[train_ind], y[valid_ind]\n    \n    model = get_model()\n    if i == 0:\n        print(model.summary())\n        \n    er = tf.keras.callbacks.EarlyStopping(monitor='val_binary_crossentropy', patience=40, restore_best_weights=True)\n    model.fit([X_train[:, 2:], X_train[:, 0].astype('int32'), X_train[:, 1].astype('int32')], y_train, \n              epochs=512, batch_size=64, validation_data=[[X_valid[:, 2:], X_valid[:, 0].astype('int32'), X_valid[:, 1].astype('int32')], y_valid], \n              verbose=0, callbacks=[er])\n    \n    preds = model.predict([X_valid[:, 2:], X_valid[:, 0].astype('int32'), X_valid[:, 1].astype('int32')])\n    sm, loss = spline_model(y_valid.flatten(), preds.flatten())\n    print(f'Fold {i}: {loss}')\n    test_pred = model.predict([X_test[:, 2:], X_test[:, 0].astype('int32'), X_test[:, 1].astype('int32')])\n    predicts.append(sm(test_pred))\n\n# Take the average probabilty on 5 folds\npredicts = np.asarray(predicts)\npredicts = np.mean(predicts, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicts.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n\nsns.distplot(predicts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = predicts\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":4}