{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hey everyone .. here is a simple usage of DNN with SK-Fold for March Madness 2020 NCAAM"},{"metadata":{},"cell_type":"markdown","source":"This note book is a follow up of this [kernel](https://www.kaggle.com/hiromoon166/2020-basic-starter-kernel)."},{"metadata":{},"cell_type":"markdown","source":"## Import Library & Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"tourney_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')\ntourney_seed = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# deleting unnecessary columns\ntourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\n\ntourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_seed(x):\n    return int(x[1:3])\n\ntourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))\ntourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge Score"},{"metadata":{"trusted":true},"cell_type":"code","source":"season_result = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"season_win_result = season_result[['Season', 'WTeamID', 'WScore']]\nseason_lose_result = season_result[['Season', 'LTeamID', 'LScore']]\n\nseason_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)\nseason_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)\n\nseason_result = pd.concat((season_win_result, season_lose_result)).reset_index(drop=True)\nseason_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()\nseason_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\n\ntourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)\ntourney_result = tourney_result.drop('TeamID', axis=1)\n\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)\ntourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True)\ntourney_win_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_lose_result = tourney_win_result.copy()\n\ntourney_lose_result['Seed1'] = tourney_win_result['Seed2']\ntourney_lose_result['Seed2'] = tourney_win_result['Seed1']\n\ntourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']\ntourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']\n\ntourney_lose_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']\ntourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']\n\ntourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']\ntourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tourney_win_result['result'] = 1\ntourney_lose_result['result'] = 0\ntourney_result = pd.concat((tourney_win_result, tourney_lose_result)).reset_index(drop=True)\ntourney_result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing testing data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))\ntest_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))\ntest_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Seed':'Seed1'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\n\ntest_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Seed':'Seed2'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\n\ntest_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Score':'ScoreT1'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\n\ntest_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')\ntest_df.rename(columns={'Score':'ScoreT2'}, inplace=True)\ntest_df = test_df.drop('TeamID', axis=1)\n\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))\ntest_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))\ntest_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']\ntest_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']\ntest_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)\ntest_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"X = tourney_result.drop('result', axis=1)\ny = tourney_result.result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler=StandardScaler()\n\n\ntest_df=scaler.fit_transform(test_df)\ntest_df=test_df.reshape(test_df.shape[0],test_df.shape[1],1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import LSTM, Dropout, MaxPool1D, Flatten, Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom tensorflow.keras import callbacks\n\nfrom sklearn.metrics import accuracy_score, roc_auc_score, auc\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics, preprocessing\nfrom tensorflow.keras import backend as K\n\ndef NN_model():\n  # model\n    model=Sequential()\n    model.add(Flatten(input_shape=(6,1)))\n\n    model.add(Dense(64, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.2))\n\n    model.add(Dense(128, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.2))\n\n    model.add(Dense(256, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.3))\n    \n    model.add(Dense(512, activation='relu'))\n    model.add(Dropout(0.3))\n\n    model.add(Dense(1,activation='sigmoid'))\n\n    model.compile(optimizer=Adam(lr=1e-3), loss='binary_crossentropy', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nNFOLDS = 50\nfolds = StratifiedKFold(n_splits=NFOLDS)\n\ncolumns = X.columns\nsplits = folds.split(X, y)\ny_preds = np.zeros(test_df.shape[0])\ny_oof = np.zeros(X.shape[0])\n\nfeature_importances = pd.DataFrame()\nfeature_importances['feature'] = columns\n  \nfor fold_n, (train_index, valid_index) in enumerate(splits):\n    \n    print('Fold:',fold_n+1)\n    X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n    \n    #----------------------------------------------------------#\n    # data preparation\n\n    X_train=scaler.fit_transform(X_train)\n    X_valid=scaler.fit_transform(X_valid)\n\n    y_train=y_train.to_numpy()\n    y_valid=y_valid.to_numpy()\n\n    X_train=X_train.reshape(X_train.shape[0],X_train.shape[1],1)\n    X_valid=X_valid.reshape(X_valid.shape[0],X_valid.shape[1],1)\n\n    #----------------------------------------------------------#\n\n    #set early stopping criteria\n    \n    es = callbacks.EarlyStopping(monitor='val_accuracy', min_delta=0.001, patience=5,verbose=1, mode='max', baseline=None, restore_best_weights=True)\n    rlr = callbacks.ReduceLROnPlateau(monitor='val_accuracy', factor=0.5,patience=3, min_lr=1e-6, mode='max', verbose=1)\n\n    model = NN_model() #model creation\n    \n    model.fit(X_train, y_train, epochs=25, batch_size=32, callbacks=[es, rlr], verbose=0,validation_data=(X_valid, y_valid))\n\n    #----------------------------------------------------------#\n    #validation\n    \n    y_pred_valid = model.predict(X_valid)\n    y_oof[valid_index] = y_pred_valid.ravel()\n    \n#     y_preds += clf.predict(test_df) / NFOLDS\n    \n    test_fold_preds = model.predict(test_df)\n    y_preds += test_fold_preds.ravel()\n    \n    print(metrics.roc_auc_score(y_valid, y_pred_valid))\n    \n    del X_train, X_valid, y_train, y_valid\n    \n    K.clear_session()\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict & Make Submission File"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds /= 50\nprint('Creating Submission File !!')\nsubmission_df = pd.read_csv('../input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')\nsubmission_df['Pred'] = y_preds\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['Pred'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**UP-VOTE this solution if you liked it ..**"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}