{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## TensorFlow Transformer \n\n****Football Match Probability Prediction - Run on GPU\n\nThis model is an adaptation of the model developed by:\n\nhttps://www.kaggle.com/cdeotte\n\nhttps://www.kaggle.com/code/cdeotte/tensorflow-transformer-0-112\n\n**Introduction**\n\n> **In case you fork/copy/like this notebook:**\n> **Upvote please**\n","metadata":{"_uuid":"7cd960c6-91d4-4ca0-a69a-9e4e520d184a","_cell_guid":"eab67375-a4a9-4e08-bb4a-3c099e21dfa9","trusted":true}},{"cell_type":"markdown","source":"## Import modules and loading the data\nFirst let's import modules, the training and the test set. The test set contains the same columns than the training set without the target.","metadata":{"_uuid":"a1e98e57-05f8-42b9-815d-023b303cd789","_cell_guid":"000b4f69-0b86-45f3-9560-8c549acb8e78","trusted":true}},{"cell_type":"code","source":"# Import libraries\n# import gc\nimport numpy as np\nimport pandas as pd\nimport datetime as dt\n#\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import RobustScaler, LabelEncoder, StandardScaler\nfrom sklearn.metrics import accuracy_score, log_loss\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, classification_report\n#\nimport tensorflow as tf\nfrom tensorflow.keras.utils import plot_model, to_categorical\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, LearningRateScheduler\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\nfrom tensorflow.keras import layers","metadata":{"_uuid":"920e6d1a-69cf-46e8-a78b-24f7fe780f73","_cell_guid":"c6c6ebe4-3bf5-4a40-8f8e-3f065ecd9e16","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#loading the data\ntrain = pd.read_csv('/kaggle/input/football-match-probability-prediction/train.csv')\ntest = pd.read_csv('/kaggle/input/football-match-probability-prediction/test.csv')\nsubmission = pd.read_csv('/kaggle/input/football-match-probability-prediction/sample_submission.csv')\ntrain_target_and_scores = pd.read_csv('../input/football-match-probability-prediction/train_target_and_scores.csv')","metadata":{"_uuid":"f9cd6673-ccc8-4e4c-bce0-c4e7e1d8e98a","_cell_guid":"e5fbcac1-440c-4d84-8f6f-6ef0f1102c70","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Parameters\nSet some parameters that will be used later.","metadata":{"_uuid":"ac3c4d94-2456-491c-b634-480be3505386","_cell_guid":"62203296-73b6-416a-b8af-83919edc6721","trusted":true}},{"cell_type":"code","source":"# Set seed\nSEED = 2022\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\n#Some parameters\nMASK = -666 # fill NA with -666 (the number of the beast)\nT_HIST = 10 # time history, last 10 games\nCLASS = 3 #number of classes (home, draw, away)\n\nDEBUG = False\n# Run on a small sample of the data\nif DEBUG:\n    train = train[:1000]","metadata":{"_uuid":"9bc24db5-b55a-4e16-87c3-ce6b4cd8a552","_cell_guid":"72e32380-622d-40ce-b35a-674120f981b1","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exclude matches with no history at date 1 - full of NA (1159 rows excluded)\ntrain.dropna(subset=['home_team_history_match_date_1'], inplace = True)\n# exclude leagues with less than 15 matches: This change has no effect on the final log loss\nif False:\n    train['league_id_count'] = train.groupby('league_id')['id'].transform('count')\n    train = train.loc[train['league_id_count'] > 15]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train: {train.shape} \\n Submission: {submission.shape}\")\ntrain.head()","metadata":{"_uuid":"f98d00e1-b8db-475f-926b-3f27fc07bc55","_cell_guid":"36d932ff-1cf7-4fa3-97c3-2657a83150a9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering\nOne of the most important parts of Machine Learning.","metadata":{"_uuid":"91794a39-6ab8-4fac-bf6a-833de1182b06","_cell_guid":"838bf7ec-34de-46d8-8de9-0c9be9cb1fea","trusted":true}},{"cell_type":"code","source":"# for cols \"date\", change to datatime \nfor col in train.filter(regex='date', axis=1).columns:\n    train[col] = pd.to_datetime(train[col])\n    test[col] = pd.to_datetime(test[col])\n\n# Some feature engineering\ndef add_features(df):\n    for i in range(1, 11): # range from 1 to 10\n        # Feat. difference of days\n        df[f'home_team_history_match_DIFF_day_{i}'] = (df['match_date'] - df[f'home_team_history_match_date_{i}']).dt.days\n        df[f'away_team_history_match_DIFF_days_{i}'] = (df['match_date'] - df[f'away_team_history_match_date_{i}']).dt.days\n    # Feat. difference of scored goals\n        df[f'home_team_history_DIFF_goal_{i}'] = df[f'home_team_history_goal_{i}'] - df[f'home_team_history_opponent_goal_{i}']\n        df[f'away_team_history_DIFF_goal_{i}'] = df[f'away_team_history_goal_{i}'] - df[f'away_team_history_opponent_goal_{i}']\n    # Feat dummy winner x loser\n        df[f'home_winner_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] > 0, 1., 0.) \n        df[f'home_loser_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] < 0, 1., 0.)\n        df[f'away_winner_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] > 0, 1., 0.)\n        df[f'away_loser_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] < 0, 1., 0.)\n    # Results: multiple nested where # away:0, draw:1, home:2\n        df[f'home_team_result_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] > 0., 2.,\n                         (np.where(df[f'home_team_history_DIFF_goal_{i}'] == 0., 1,\n                                   np.where(df[f'home_team_history_DIFF_goal_{i}'].isna(), np.nan, 0))))\n        df[f'away_team_result_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] > 0., 2.,\n                         (np.where(df[f'away_team_history_DIFF_goal_{i}'] == 0., 1.,\n                                   np.where(df[f'away_team_history_DIFF_goal_{i}'].isna(), np.nan, 0.))))\n    # Feat. difference of rating (\"modified\" ELO RATING)\n        df[f'home_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'home_team_history_opponent_rating_{i}']-df[f'home_team_history_rating_{i}'])/10))\n        df[f'away_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'away_team_history_opponent_rating_{i}']-df[f'away_team_history_rating_{i}'])/10))\n        df[f'home_away_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'away_team_history_rating_{i}']-df[f'home_team_history_rating_{i}'])/10))\n        # df[f'home_away_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'away_team_history_rating_{i}']- (2*df[f'home_team_history_is_play_home_{i}'] + df[f'home_team_history_rating_{i}']))/10))\n    # Feat. same coach id\n        df[f'home_team_history_SAME_coaX_{i}'] = np.where(df['home_team_coach_id']==df[f'home_team_history_coach_{i}'],1,0)\n        df[f'away_team_history_SAME_coaX_{i}'] = np.where(df['away_team_coach_id']==df[f'away_team_history_coach_{i}'],1,0) \n    # Feat. same league id\n        df[f'home_team_history_SAME_leaG_{i}'] = np.where(df['league_id']==df[f'home_team_history_league_id_{i}'],1,0)\n        df[f'away_team_history_SAME_leaG_{i}'] = np.where(df['league_id']==df[f'away_team_history_league_id_{i}'],1,0) \n    # Fill NA with -666\n    # df.fillna(MASK, inplace = True)\n    return df\n\ntrain = add_features(train)\ntest = add_features(test)","metadata":{"_uuid":"daed24ca-8f78-4419-b19f-3b97d78f29f7","_cell_guid":"fa41a07c-c823-4716-8666-21f3f8cc12ed","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scaling and Reshape","metadata":{"_uuid":"0fa67e85-50f7-43af-a9dc-b11c05bcbf8c","_cell_guid":"878d27a4-831a-4a9f-9c9b-275409afc69b","trusted":true}},{"cell_type":"code","source":"# save targets\n# train_id = train['id'].copy()\ntrain_y = train['target'].copy()\n#keep only some features\ntrain_x = train.drop(['target', 'home_team_name', 'away_team_name'], axis=1) #, inplace=True) # is_cup EXCLUDED\n# Exclude all date, league, coach columns\ntrain_x.drop(train.filter(regex='date').columns, axis=1, inplace = True)\ntrain_x.drop(train.filter(regex='league').columns, axis=1, inplace = True)\ntrain_x.drop(train.filter(regex='coach').columns, axis=1, inplace = True)\n#\n# Test set\n# test_id = test['id'].copy()\ntest_x = test.drop(['home_team_name', 'away_team_name'], axis=1)#, inplace=True) # is_cup EXCLUDED\n# Exclude all date, league, coach columns\ntest_x.drop(test.filter(regex='date').columns, axis=1, inplace = True)\ntest_x.drop(test.filter(regex='league').columns, axis=1, inplace = True)\ntest_x.drop(test.filter(regex='coach').columns, axis=1, inplace = True)","metadata":{"_uuid":"7b65ce25-5a33-4fb8-b04b-0f288ea3d9bb","_cell_guid":"fe6715ba-250e-4a67-bc76-f046ba57b793","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Target, train and test shape\nprint(f\"Target: {train_y.shape} \\n Train shape: {train_x.shape} \\n Test: {test_x.shape}\")\nprint(f\"Column names: {list(train_x.columns)}\")","metadata":{"_uuid":"9c748333-e838-48d8-88f0-33ef0051e57e","_cell_guid":"9ab58872-8c97-4c9a-a110-a55588e0f29f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x.head()","metadata":{"_uuid":"e45561e1-2c45-4bb8-8b20-70aa386fc82a","_cell_guid":"9dda4bc6-19b1-40f4-bab6-ca33dae90f12","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Store feature names\n# feature_names = list(train.columns)\n# Pivot dataframe to create an input array for the LSTM network\nfeature_groups = [\"home_team_history_is_play_home\", \"home_team_history_is_cup\",\n    \"home_team_history_goal\", \"home_team_history_opponent_goal\",\n    \"home_team_history_rating\", \"home_team_history_opponent_rating\",  \n    \"away_team_history_is_play_home\", \"away_team_history_is_cup\",\n    \"away_team_history_goal\", \"away_team_history_opponent_goal\",\n    \"away_team_history_rating\", \"away_team_history_opponent_rating\",  \n    \"home_team_history_match_DIFF_day\", \"away_team_history_match_DIFF_days\",\n    \"home_team_history_DIFF_goal\",\"away_team_history_DIFF_goal\",\n    \"home_team_history_ELO_rating\",\"away_team_history_ELO_rating\",\n    \"home_away_team_history_ELO_rating\",\n    \"home_team_history_SAME_coaX\", \"away_team_history_SAME_coaX\",\n    \"home_team_history_SAME_leaG\", \"away_team_history_SAME_leaG\",\n    \"home_team_result\", \"away_team_result\",\n    \"home_winner\", \"home_loser\", \"away_winner\", \"away_loser\"\n                 ]      \n# Pivot dimension (id*features) x time_history\ntrain_x_pivot = pd.wide_to_long(train_x, stubnames=feature_groups, \n                i=['id','is_cup'], j='time', sep='_', suffix='\\d+')\ntest_x_pivot = pd.wide_to_long(test_x, stubnames=feature_groups, \n                i=['id','is_cup'], j='time', sep='_', suffix='\\d+')\n#\nprint(f\"Train pivot shape: {train_x_pivot.shape}\")  \nprint(f\"Test pivot shape: {test_x_pivot.shape}\")","metadata":{"_uuid":"c316690b-dbb6-45f2-882c-18bcb932af61","_cell_guid":"46437c51-e209-4f27-8d53-1bf0e8547fc4","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create columns based on index\ntrain_x_pivot = train_x_pivot.reset_index()\ntest_x_pivot = test_x_pivot.reset_index()\n# Deal with the is_cup feature\n# There are NA in 'is_cup'\ntrain_x_pivot=train_x_pivot.fillna({'is_cup':False})\ntrain_x_pivot['is_cup'] = pd.get_dummies(train_x_pivot['is_cup'], drop_first=True)\n#\ntest_x_pivot=test_x_pivot.fillna({'is_cup':False})\ntest_x_pivot['is_cup']= pd.get_dummies(test_x_pivot['is_cup'], drop_first=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_pivot.head(20)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_pivot.head(20)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Changing the sequence of time from 1...10 to 10...1 improve the model?\n# bidirectional LSTM is used\n\nINV = True\nif INV:\n    # Trying to keep the same id order\n    train_x_pivot.sort_values(by=['time'], inplace = True, ascending=False)\n    # Merge and drop columns\n    train_x_pivot = pd.merge(train_x['id'], train_x_pivot, on=\"id\")#.drop(['id', 'time'], axis = 1)\n    # Test\n    test_x_pivot.sort_values(by=['time'], inplace = True, ascending=False)\n    test_x_pivot = pd.merge(test_x['id'], test_x_pivot, on=\"id\")#.drop(['id', 'time'], axis = 1)\n    \n    # x_test_pivot = x_test_pivot.reset_index()\n    # x_test_pivot['time'] = (T_HIST + 1) - x_test_pivot['time']\n    # x_test_pivot.sort_values(by=['time'], inplace = True)\n    # x_test = pd.merge(test_id, x_test_pivot, on=\"id\").drop(['id', 'time'], axis = 1)\n    # x_test = x_test.to_numpy().reshape(-1, T_HIST, x_test.shape[-1])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature engineering again!!! Group by id, stats over time\ndef add_features_II(df):\n    df['ewm_home_away_team_history_ELO_rating_mean'] = df.groupby('id')['home_away_team_history_ELO_rating'].ewm(com=0.5).mean().reset_index(level=0,drop=True)\n    df['ewm_home_away_team_history_ELO_rating_std'] = df.groupby('id')['home_away_team_history_ELO_rating'].ewm(com=0.5).std().reset_index(level=0,drop=True)\n    # goals\n    df['home_team_history_DIFF_goal_ewm'] = df.groupby('id')['home_team_history_DIFF_goal'].ewm(com=0.5).mean().reset_index(level=0,drop=True)# .cumsum()\n    df['away_team_history_DIFF_goal_ewm'] = df.groupby('id')['away_team_history_DIFF_goal'].ewm(com=0.5).mean().reset_index(level=0,drop=True)#.cumsum()\n    #\n    # df['home_team_hist_goal_csum'] = df.groupby('id')['home_team_history_goal'].cumsum()\n    # df['home_team_hist_opp_goal_csum'] = df.groupby('id')['home_team_history_opponent_goal'].cumsum()\n    # df['away_team_hist_goal_csum'] = df.groupby('id')['away_team_history_goal'].cumsum()\n    # df['away_team_hist_opp_goal_csum'] = df.groupby('id')['away_team_history_opponent_goal'].cumsum()\n    # rating\n    # df['home_team_hist_rat_mean'] = df.groupby('id')['home_team_history_rating'].transform('mean')\n    # df['away_team_hist_rat_mean'] = df.groupby('id')['away_team_history_rating'].transform('mean')\n    # df['home_away_rat_elo'] = 1/(1+10**((df['away_team_hist_rat_mean']-df['home_team_hist_rat_mean'])/10))\n    # Result (%)\n    # df['home_team_result_mean'] = df.groupby('id')['home_team_result'].transform('mean')\n    # df['away_team_result_mean'] = df.groupby('id')['away_team_result'].transform('mean')\n    #\n    # df['home_team_result_perc'] = df.groupby('id')['home_team_result'].cumsum()\n    # df['away_team_result_perc'] = df.groupby('id')['away_team_result'].cumsum()\n    # df['home_team_result_perc'] = df['home_team_result_perc']/(df['time']*2)\n    # df['away_team_result_perc'] = df['away_team_result_perc']/(df['time']*2)\n    # Lags result if INV = True\n    #\n    df['home_away_team_history_ELO_rating_lag1'] = df.groupby('id')['home_away_team_history_ELO_rating'].shift(1)\n    df['home_away_team_history_ELO_rating_lag2'] = df.groupby('id')['home_away_team_history_ELO_rating'].shift(2)\n    #\n    # dummies results\n    # df = pd.get_dummies(df['home_team_result'])\n    # df = pd.get_dummies(df['away_team_result'])\n    # df['one'] = 1\n    # df['count'] = df.groupby('id').isna()\n    \n    return df\n\n# train_x_pivot = add_features_II(train_x_pivot)\n# test_x_pivot = add_features_II(test_x_pivot)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_pivot.head(20)\ntest_x_pivot.head(20)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train_x_pivot.drop(['id', 'time'], axis=1).copy() \nx_test = test_x_pivot.drop(['id', 'time'], axis=1).copy()\n# Fill NA with median\nfill_mean = True\nif fill_mean:\n    x_train = np.where(np.isnan(x_train), np.nanmean(x_train, axis=0), x_train)\n    x_test = np.where(np.isnan(x_test), np.nanmean(x_test, axis=0), x_test)\n'''\nfill_median = True\nif fill_median:\n    x_train = np.where(np.isnan(x_train), np.nanmedian(x_train, axis=0), x_train)\n    x_test = np.where(np.isnan(x_test), np.nanmedian(x_test, axis=0), x_test)\n'''\n# Scale features using statistics that are robust to outliers\n# RS = RobustScaler()\nSS = StandardScaler()\nx_train = SS.fit_transform(x_train)\nx_test = SS.transform(x_test)\n# Reshape \nx_train = x_train.reshape(-1, T_HIST, x_train.shape[-1])\nx_test = x_test.reshape(-1, T_HIST, x_test.shape[-1])\n\nif False:\n    # Fill NA with MASK\n    x_train = np.nan_to_num(x_train, nan=MASK)\n    x_test = np.nan_to_num(x_test, nan=MASK)\n\n# Back to pandas.dataframe\n# x_train = pd.DataFrame(train, columns=feature_names)\n# x_train = pd.concat([train_id, x_train], axis = 1)\n#\n# x_test = pd.DataFrame(test, columns=feature_names)\n# x_test = pd.concat([test_id, x_test], axis = 1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train array shape: {x_train.shape} \\nTest array shape: {x_test.shape}\")","metadata":{"_uuid":"9adbb500-7f82-41be-aa8f-f19c9291c4e9","_cell_guid":"bedd04b7-cf54-4289-b3c9-b6086f90a580","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Deal with targets\n# encode class values as integers\nencoder = LabelEncoder()\nencoder.fit(train_y)\nencoded_y = encoder.transform(train_y)\n# convert integers to dummy variables (i.e. one hot encoded)\ndummy_y = to_categorical(encoded_y)\n# \nprint(encoded_y.shape)\nprint(dummy_y.shape)","metadata":{"_uuid":"129f90f7-7744-488c-9e61-b49bb2c90992","_cell_guid":"40d047c0-ca6f-4bdc-af0f-17e0d1f843f5","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# encoding away: 0 draw: 1 home: 2 \nprint(encoded_y[:10,])\n# Order: away, draw, home\nprint(dummy_y[:10,])","metadata":{"_uuid":"f66ba6cb-16e0-4560-a345-e5a38b5ed82c","_cell_guid":"6a9dcfac-a3a7-40a2-b7c1-c515319b7e0c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Setup GPU\nBelow declare whether to use 1 GPU or multiple GPU. (Change CUDA_VISIBLE_DEVICES to use more GPUs). I am not aware of the impact on speed.","metadata":{}},{"cell_type":"code","source":"import os\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n\n# USE MULTIPLE GPUS\nif os.environ[\"CUDA_VISIBLE_DEVICES\"].count(',') == 0:\n    gpu_strategy = tf.distribute.get_strategy()\n    print('single strategy')\nelse:\n    gpu_strategy = tf.distribute.MirroredStrategy()\n    print('multiple strategy')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make a model\n**Attention Is All You Need**\nAshish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser, Illia Polosukhin\n\nThe dominant sequence transduction models are based on complex recurrent or convolutional neural networks in an encoder-decoder configuration. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely...","metadata":{"_uuid":"f0fd77a2-0d9c-4fba-ac6d-f879a553bf4c","_cell_guid":"51235a13-fea9-47fd-a2b9-caeccbd0b67f","trusted":true}},{"cell_type":"code","source":"class TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, feat_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = tf.keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"gelu\"), layers.Dense(feat_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)","metadata":{"_uuid":"77fb7a98-c7fb-40e9-87d6-d629076cc8fc","_cell_guid":"0b944518-27e9-46ed-a31a-408b675bdd1b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_dim = 8 # 32 # x_train.shape[-1] + 32\nembed_dim = 4 # 16 # 64  # Embedding size for attention\nnum_heads = 2 # 4 # 8  # Number of attention heads\nff_dim = 8 # 16 # 128  # Hidden layer size in feed forward network inside transformer\ndropout_rate = 0.5 # 0.2 # 0.0\nnum_blocks = 2 # 12# 4 \nskip = .5\n\ndef build_model():\n    inputs = layers.Input(shape=x_train.shape[-2:])\n        \n    # \"EMBEDDING LAYER\"\n    x = layers.Dense(feat_dim)(inputs)\n    x = layers.LayerNormalization(epsilon=1e-6)(x)\n    \n    # TRANSFORMER BLOCKS\n    for k in range(num_blocks):\n        x_old = x\n        transformer_block = TransformerBlock(embed_dim, feat_dim, num_heads, ff_dim, dropout_rate)\n        x = transformer_block(x)\n        x = skip*x + (1-skip)*x_old # SKIP CONNECTION\n    \n    # REGRESSION HEAD\n    # x = layers.Dense(16, activation=\"selu\")(x) # 128\n    # x = layers.Dropout(dropout_rate)(x)\n    # outputs = layers.Dense(3, activation=\"linear\")(x)\n    x = layers.Flatten()(x)\n    outputs = layers.Dense(CLASS, activation='softmax')(x)\n    model = Model(inputs=inputs, outputs=outputs)\n        \n    return model","metadata":{"_uuid":"8f733748-8539-4dff-9db7-88edd1e41bdf","_cell_guid":"da442908-1692-4121-b108-ebd4141c4d4f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Another try, model number 2\nif False:\n    embed_dim = 16 # 32  # Embedding size for each token\n    num_heads = 2  # Number of attention heads\n    ff_dim = 8 # 32  # Hidden layer size in feed forward network inside transformer\n\n    def build_model_2():\n        # inputs = layers.Input(shape=(maxlen,))\n        # embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\n        # x = embedding_layer(inputs)\n        inputs = layers.Input(shape=x_train.shape[-2:])\n        # \"EMBEDDING LAYER\"\n        x = layers.Dense(feat_dim)(inputs)\n        x = layers.LayerNormalization(epsilon=1e-6)(x)\n        transformer_block = TransformerBlock(embed_dim, feat_dim, num_heads, ff_dim, dropout_rate)\n        x = transformer_block(x)\n        # x = layers.GlobalAveragePooling1D()(x)\n        # x = layers.Dropout(dropout_rate)(x)\n        x = layers.Dense(8, activation=\"relu\")(x)\n        x = layers.Dropout(dropout_rate)(x)\n        x = layers.Flatten()(x)\n        outputs = layers.Dense(CLASS, activation='softmax')(x)\n        model = Model(inputs=inputs, outputs=outputs)\n        return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This learning rate scheduler is not necessary...\n'''\n\nimport math \nimport matplotlib.pyplot as plt\n\nLR_START = 1e-6\nLR_MAX = 6e-4\nLR_MIN = 1e-6\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 0\nEPOCHS = 420\nSTEPS = [60,120,240]\n\n\ndef lrfn(epoch):\n    if epoch<STEPS[0]:\n        epoch2 = epoch\n        EPOCHS2 = STEPS[0]\n    elif epoch<STEPS[0]+STEPS[1]:\n        epoch2 = epoch-STEPS[0]\n        EPOCHS2 = STEPS[1]\n    elif epoch<STEPS[0]+STEPS[1]+STEPS[2]:\n        epoch2 = epoch-STEPS[0]-STEPS[1]\n        EPOCHS2 = STEPS[2]\n    \n    if epoch2 < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch2 + LR_START\n    elif epoch2 < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        decay_total_epochs = EPOCHS2 - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n        decay_epoch_index = epoch2 - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        phase = math.pi * decay_epoch_index / decay_total_epochs\n        cosine_decay = 0.5 * (1 + math.cos(phase))\n        lr = (LR_MAX - LR_MIN) * cosine_decay + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\nlr_y = [lrfn(x) for x in rng]\nplt.figure(figsize=(10, 4))\nplt.plot(rng, lr_y, '-o')\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\". \\\n          format(lr_y[0], max(lr_y), lr_y[-1]))\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\nplt.xlabel('Epoch',size=14)\nplt.ylabel('Learning Rate',size=14)\nplt.show()\n'''","metadata":{"_uuid":"6d6ccc48-5945-415d-8cf8-c35b7335e743","_cell_guid":"e648e1b0-6967-4cfb-af34-173dc29dea76","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Choose your model\nmodel = build_model()\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Results\n# Epoch 40/300\n# 172/172 [==============================] - 2s 11ms/step - loss: 0.9887 - accuracy: 0.5155 - val_loss: 1.0091 - val_accuracy: 0.4989\n#2\n# Epoch 125/300\n# 172/172 [==============================] - 1s 7ms/step - loss: 1.0029 - accuracy: 0.5039 - val_loss: 1.0042 - val_accuracy: 0.5016\n#3\n# Epoch 00300: LearningRateScheduler setting learning rate to 0.00030246841051434123.\n# 172/172 [==============================] - 1s 7ms/step - loss: 1.0027 - accuracy: 0.5049 - val_loss: 1.0037 - val_accuracy: 0.5003\n# SEM  # \"home_winner\", \"home_loser\", \"away_winner\", \"away_loser\"\n#Epoch 144/300\n#172/172 [==============================] - 1s 7ms/step - loss: 1.0044 - accuracy: 0.5024 - val_loss: 1.0047 - val_accuracy: 0.5014\n# SEM \"home_team_history_SAME_coaX\", \"away_team_history_SAME_coaX\",\n    # \"home_team_history_SAME_leaG\", \"away_team_history_SAME_leaG\",\n# Epoch 130/300\n# 172/172 [==============================] - 1s 7ms/step - loss: 1.0072 - accuracy: 0.4998 - val_loss: 1.0080 - val_accuracy: 0.4998\n# Tudo\n# Epoch 136/300\n# 172/172 [==============================] - 1s 7ms/step - loss: 1.0026 - accuracy: 0.5033 - val_loss: 1.0029 - val_accuracy: 0.5007\n# 2 transformers, skip = 50%\n# Epoch 135/300\n# 172/172 [==============================] - 2s 10ms/step - loss: 1.0022 - accuracy: 0.5038 - val_loss: 1.0024 - val_accuracy: 0.5041\n#\n# 1 transform, skip = 50%\n# Epoch 166/300\n# 172/172 [==============================] - 2s 12ms/step - loss: 1.0029 - accuracy: 0.5036 - val_loss: 1.0028 - val_accuracy: 0.5040\n# 4 transform skip 50%\n# Epoch 116/300\n# 172/172 [==============================] - 3s 20ms/step - loss: 1.0012 - accuracy: 0.5039 - val_loss: 1.0025 - val_accuracy: 0.5039\n#\n# 12 transform 50% skip\n# Epoch 99/300\n# 172/172 [==============================] - 7s 42ms/step - loss: 1.0001 - accuracy: 0.5047 - val_loss: 1.0038 - val_accuracy: 0.5031\n#\n# INCLUDING FEATURES II\n# Epoch 102/300\n# 172/172 [==============================] - 1s 7ms/step - loss: 1.0047 - accuracy: 0.5020 - val_loss: 1.0042 - val_accuracy: 0.4996\n\n#### MODEL 2\n\n# 3 model_2\n# Epoch 103/300\n# 172/172 [==============================] - 1s 7ms/step - loss: 1.0182 - accuracy: 0.4958 - val_loss: 1.0085 - val_accuracy: 0.4981\n# 4 model_2\n# Epoch 89/300\n# 172/172 [==============================] - 1s 7ms/step - loss: 1.0093 - accuracy: 0.4996 - val_loss: 1.0052 - val_accuracy: 0.4974\n# 5 model 2\n# Epoch 00179: LearningRateScheduler setting learning rate to 1.1043632102797669e-06.\n# 172/172 [==============================] - 1s 7ms/step - loss: 1.0093 - accuracy: 0.4996 - val_loss: 1.0045 - val_accuracy: 0.4971\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCH = 300\nBATCH_SIZE = 512 # 1024 # 256\n# 1024\n# Epoch 165/300\n# 86/86 [==============================] - 1s 8ms/step - loss: 1.0030 - accuracy: 0.5027 - val_loss: 1.0049 - val_accuracy: 0.5030\n# 256\n# Epoch 169/300\n# 344/344 [==============================] - 3s 9ms/step - loss: 1.0018 - accuracy: 0.5047 - val_loss: 1.0039 - val_accuracy: 0.5039\nNUM_FOLDS = 5\nVERBOSE = 1\nPATIENCE = EPOCH // 10\nFIRST_FOLD_ONLY = False\n\ntest_preds = []\n\nwith gpu_strategy.scope():\n    kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=SEED)\n    for fold, (train_idx, test_idx) in enumerate(kf.split(x_train, dummy_y)):\n        print('-'*15, '>', f'Fold {fold+1}/{NUM_FOLDS}', '<', '-'*15)\n        X_train, X_valid = x_train[train_idx], x_train[test_idx]\n        Y_train, Y_valid = dummy_y[train_idx], dummy_y[test_idx]\n        # Model\n        model = build_model()\n        # It is a multi-class classification problem, categorical_crossentropy is used as the loss function.\n        model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\",\n                     metrics=[\"accuracy\"])\n        #\n        es = EarlyStopping(monitor='val_loss', patience=PATIENCE, verbose=0, mode='min',\n                           restore_best_weights=True)\n        lr = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.25, patience=10, verbose=0)\n        #\n        model.fit(X_train, Y_train, \n                  validation_data=(X_valid, Y_valid), \n                  epochs=EPOCH,\n                  verbose=VERBOSE,\n                  batch_size=BATCH_SIZE,  \n                  callbacks=[lr, es])\n        # Model validation    \n        y_true = Y_valid.squeeze()\n        y_pred = model.predict(X_valid, batch_size=BATCH_SIZE).squeeze()\n        score1 = log_loss(y_true, y_pred)\n        print(f\"Fold-{fold+1} | OOF LogLoss Score: {score1}\")\n        # Predictions\n        test_preds.append(model.predict(x_test).squeeze())\n        # test_preds.append(model.predict(x_test, batch_size=BATCH_SIZE).squeeze().reshape(-1, 1).squeeze())\n        if FIRST_FOLD_ONLY: break","metadata":{"_uuid":"677f1522-558d-4ca0-8c08-205590b91045","_cell_guid":"8785689a-4fac-4269-9be0-fb80652f93b1","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds = np.argmax(y_pred, axis=1) # preds = np.where(y_pred > 0.5, 1, 0)\n# report = classification_report(y_true, preds)\n# print(report)\n# cm = confusion_matrix(y_true, preds)\n# # encoding away: 0 draw: 1 home:2\n# disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels={'away', 'draw','home',})\n# disp.plot()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit the model and a make submission\nOnce the model is fitted we make the **probabilities prediction**. Then we make the submission dataframe with **4 columns**. The columns home, away and draw contain probability while the column id contains the match id.","metadata":{"_uuid":"ea635cb3-5380-4259-ad5f-fd94e9606f26","_cell_guid":"0c123082-ce45-4f81-841d-eb90adea1783","trusted":true}},{"cell_type":"code","source":"predictions = sum(test_preds)/NUM_FOLDS \n\n# away, draw, home\nsubmission = pd.DataFrame(predictions,columns=['away', 'draw', 'home'])\n\n# Round\nround_num = False\nif round_num:\n    submission = submission.round(2)\n    submission['draw'] = 1 - (submission['home'] + submission['away'])  \n    \n#do not forget the id column\nsubmission['id'] = test[['id']]\n\n#submit!\nsubmission[['id', 'home', 'away', 'draw']].to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[['id', 'home', 'away', 'draw']].head()","metadata":{"_uuid":"a5aeb080-c652-4e2c-9476-e662c51ff731","_cell_guid":"a7b164c3-f7d6-47a8-b6a6-b84ebb382b4d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion\n**Good luck!**\n\nReport any error that you will probably find...","metadata":{"_uuid":"002cd78f-9689-47ca-a8fe-2e28eba99b1d","_cell_guid":"4a755058-46c1-4ab7-9693-2381814c1bc0","trusted":true}}]}