{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# > **Competition Description**\n> Football has been at the heart of data science for more than a decade. If today's algorithms focus on event detection, player style, or team analysis, predicting the results of a match stays an open challenge.\n> \n> Predicting the outcomes of a match between two teams depends mostly (but not only) on their current form. The form of a team can be viewed as their recent sequence of results versus the other teams. So match probabilities between two teams can be different given their calendar.\n> \n> This competition is about predicting the probabilities of more than 150000 match outcomes using the recent sequence of 10 matches of the teams.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-06T23:18:03.199411Z","iopub.execute_input":"2022-05-06T23:18:03.199736Z","iopub.status.idle":"2022-05-06T23:18:03.231925Z","shell.execute_reply.started":"2022-05-06T23:18:03.199649Z","shell.execute_reply":"2022-05-06T23:18:03.230915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Dataset","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime as dt\n#\nfrom sklearn.model_selection import KFold, StratifiedKFold, TimeSeriesSplit\nfrom sklearn.preprocessing import RobustScaler, LabelEncoder\nfrom sklearn.metrics import accuracy_score, log_loss\n#\nimport tensorflow as tf\nfrom tensorflow.keras.utils import plot_model, to_categorical\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, LearningRateScheduler\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\nfrom tensorflow.keras import layers\n\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:18:16.183696Z","iopub.execute_input":"2022-05-06T23:18:16.184022Z","iopub.status.idle":"2022-05-06T23:18:23.963801Z","shell.execute_reply.started":"2022-05-06T23:18:16.183986Z","shell.execute_reply":"2022-05-06T23:18:23.962753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/football-match-probability-prediction/train.csv')\ntest = pd.read_csv('/kaggle/input/football-match-probability-prediction/test.csv')\nsubmission = pd.read_csv('/kaggle/input/football-match-probability-prediction/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:18:26.827099Z","iopub.execute_input":"2022-05-06T23:18:26.827382Z","iopub.status.idle":"2022-05-06T23:18:37.265571Z","shell.execute_reply.started":"2022-05-06T23:18:26.827353Z","shell.execute_reply":"2022-05-06T23:18:37.26479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Note :: Feature Engg and some code snippets in this notebook has been inpired by https://www.kaggle.com/code/seraquevence/ensemble-football-prob-prediction-lstm-lgbm-v01","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:50:11.124236Z","iopub.execute_input":"2022-05-06T22:50:11.124692Z","iopub.status.idle":"2022-05-06T22:50:11.149475Z","shell.execute_reply.started":"2022-05-06T22:50:11.12461Z","shell.execute_reply":"2022-05-06T22:50:11.148835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for cols \"date\", change to datatime \nfor col in train.filter(regex='date', axis=1).columns:\n    train[col] = pd.to_datetime(train[col])\n    test[col] = pd.to_datetime(test[col])","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:42:25.919789Z","iopub.execute_input":"2022-05-06T20:42:25.920323Z","iopub.status.idle":"2022-05-06T20:42:27.652651Z","shell.execute_reply.started":"2022-05-06T20:42:25.920283Z","shell.execute_reply":"2022-05-06T20:42:27.651948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:48:17.114313Z","iopub.execute_input":"2022-05-06T21:48:17.114577Z","iopub.status.idle":"2022-05-06T21:48:17.143954Z","shell.execute_reply.started":"2022-05-06T21:48:17.114548Z","shell.execute_reply":"2022-05-06T21:48:17.143282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.league_name.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:42:27.687121Z","iopub.execute_input":"2022-05-06T20:42:27.687678Z","iopub.status.idle":"2022-05-06T20:42:27.705883Z","shell.execute_reply.started":"2022-05-06T20:42:27.687627Z","shell.execute_reply":"2022-05-06T20:42:27.705181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"* Calculate historical match day difference\n* Calculate historical match goal difference\n* Check Winner\n* Check Coach\n* Check League\n* Calculate ELO rating","metadata":{}},{"cell_type":"code","source":"def add_features(df):\n    for i in range(1, 11): # range from 1 to 10\n        # Feat. difference of days\n        df[f'home_team_history_match_DIFF_days_{i}'] = (df['match_date'] - df[f'home_team_history_match_date_{i}']).dt.days\n        df[f'away_team_history_match_DIFF_days_{i}'] = (df['match_date'] - df[f'away_team_history_match_date_{i}']).dt.days\n    # Feat. difference of scored goals\n        df[f'home_team_history_DIFF_goal_{i}'] = df[f'home_team_history_goal_{i}'] - df[f'home_team_history_opponent_goal_{i}']\n        df[f'away_team_history_DIFF_goal_{i}'] = df[f'away_team_history_goal_{i}'] - df[f'away_team_history_opponent_goal_{i}']\n    # Feat dummy winner x loser\n        df[f'home_winner_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] > 0, 1., 0.) \n        df[f'home_loser_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] < 0, 1., 0.)\n        df[f'away_winner_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] > 0, 1., 0.)\n        df[f'away_loser_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] < 0, 1., 0.)\n    # Results: multiple nested where # away:0, draw:1, home:2\n        df[f'home_team_result_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] > 0., 2.,\n                         (np.where(df[f'home_team_history_DIFF_goal_{i}'] == 0., 1,\n                                   np.where(df[f'home_team_history_DIFF_goal_{i}'].isna(), np.nan, 0))))\n        df[f'away_team_result_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] > 0., 2.,\n                         (np.where(df[f'away_team_history_DIFF_goal_{i}'] == 0., 1.,\n                                   np.where(df[f'away_team_history_DIFF_goal_{i}'].isna(), np.nan, 0.))))\n    # Feat. difference of rating (\"modified\" ELO RATING)\n        df[f'home_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'home_team_history_opponent_rating_{i}']-df[f'home_team_history_rating_{i}'])/400))\n        df[f'away_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'away_team_history_opponent_rating_{i}']-df[f'away_team_history_rating_{i}'])/400))\n        df[f'home_away_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'away_team_history_rating_{i}']-df[f'home_team_history_rating_{i}'])/400))\n    # Feat. same coach id\n        df[f'home_team_history_SAME_coaX_{i}'] = np.where(df['home_team_coach_id']==df[f'home_team_history_coach_{i}'],1,0)\n        df[f'away_team_history_SAME_coaX_{i}'] = np.where(df['away_team_coach_id']==df[f'away_team_history_coach_{i}'],1,0) \n    # Feat. same league id\n        df[f'home_team_history_SAME_leaG_{i}'] = np.where(df['league_id']==df[f'home_team_history_league_id_{i}'],1,0)\n        df[f'away_team_history_SAME_leaG_{i}'] = np.where(df['league_id']==df[f'away_team_history_league_id_{i}'],1,0) \n    return df\ntrain = add_features(train)\ntest = add_features(test)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:42:27.707292Z","iopub.execute_input":"2022-05-06T20:42:27.707957Z","iopub.status.idle":"2022-05-06T20:42:28.351075Z","shell.execute_reply.started":"2022-05-06T20:42:27.707922Z","shell.execute_reply":"2022-05-06T20:42:28.350358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save targets\n# train_id = train['id'].copy()\ntrain_y = train['target'].copy()\n#keep only some features\ntrain_x = train.drop(['target', 'home_team_name', 'away_team_name'], axis=1) #, inplace=True) # is_cup EXCLUDED\n# Exclude all date, league, coach columns\ntrain_x.drop(train.filter(regex='date').columns, axis=1, inplace = True)\ntrain_x.drop(train.filter(regex='league').columns, axis=1, inplace = True)\ntrain_x.drop(train.filter(regex='coach').columns, axis=1, inplace = True)\n#\n# Test set\n# test_id = test['id'].copy()\ntest_x = test.drop(['home_team_name', 'away_team_name'], axis=1)#, inplace=True) # is_cup EXCLUDED\n# Exclude all date, league, coach columns\ntest_x.drop(test.filter(regex='date').columns, axis=1, inplace = True)\ntest_x.drop(test.filter(regex='league').columns, axis=1, inplace = True)\ntest_x.drop(test.filter(regex='coach').columns, axis=1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:48:28.94784Z","iopub.execute_input":"2022-05-06T21:48:28.948207Z","iopub.status.idle":"2022-05-06T21:48:29.496033Z","shell.execute_reply.started":"2022-05-06T21:48:28.948173Z","shell.execute_reply":"2022-05-06T21:48:29.495283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_x.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:48:30.954813Z","iopub.execute_input":"2022-05-06T21:48:30.955096Z","iopub.status.idle":"2022-05-06T21:48:30.961215Z","shell.execute_reply.started":"2022-05-06T21:48:30.955064Z","shell.execute_reply":"2022-05-06T21:48:30.959633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_x.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:48:32.268281Z","iopub.execute_input":"2022-05-06T21:48:32.269826Z","iopub.status.idle":"2022-05-06T21:48:32.296271Z","shell.execute_reply.started":"2022-05-06T21:48:32.26978Z","shell.execute_reply":"2022-05-06T21:48:32.295526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_groups = [\"home_team_history_is_play_home\", \"home_team_history_is_cup\",\n    \"home_team_history_goal\", \"home_team_history_opponent_goal\",\n    \"home_team_history_rating\", \"home_team_history_opponent_rating\",  \n    \"away_team_history_is_play_home\", \"away_team_history_is_cup\",\n    \"away_team_history_goal\", \"away_team_history_opponent_goal\",\n    \"away_team_history_rating\", \"away_team_history_opponent_rating\",  \n    \"home_team_history_match_DIFF_days\", \"away_team_history_match_DIFF_days\",\n    \"home_team_history_DIFF_goal\",\"away_team_history_DIFF_goal\",\n    \"home_team_history_ELO_rating\",\"away_team_history_ELO_rating\",\n    \"home_away_team_history_ELO_rating\",\n    \"home_team_history_SAME_coaX\", \"away_team_history_SAME_coaX\",\n    \"home_team_history_SAME_leaG\", \"away_team_history_SAME_leaG\",\n    \"home_team_result\", \"away_team_result\",\n    \"home_winner\", \"home_loser\", \"away_winner\", \"away_loser\"]      \n\ntrain_x_pivot = pd.wide_to_long(train_x, stubnames=feature_groups, \n                i=['id','is_cup'], j='time', sep='_', suffix='\\d+')\ntest_x_pivot = pd.wide_to_long(test_x, stubnames=feature_groups, \n                i=['id','is_cup'], j='time', sep='_', suffix='\\d+')","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:48:35.126483Z","iopub.execute_input":"2022-05-06T21:48:35.127017Z","iopub.status.idle":"2022-05-06T21:49:46.066849Z","shell.execute_reply.started":"2022-05-06T21:48:35.126977Z","shell.execute_reply":"2022-05-06T21:49:46.06606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_x_pivot = train_x_pivot.reset_index()\ntest_x_pivot = test_x_pivot.reset_index()\n\n\n# Encode Is_cup column\ntrain_x_pivot=train_x_pivot.fillna({'is_cup':False})\ntrain_x_pivot['is_cup'] = pd.get_dummies(train_x_pivot['is_cup'], drop_first=True)\n\ntest_x_pivot=test_x_pivot.fillna({'is_cup':False})\ntest_x_pivot['is_cup']= pd.get_dummies(test_x_pivot['is_cup'], drop_first=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:49:46.068595Z","iopub.execute_input":"2022-05-06T21:49:46.069067Z","iopub.status.idle":"2022-05-06T21:49:46.760865Z","shell.execute_reply.started":"2022-05-06T21:49:46.069025Z","shell.execute_reply":"2022-05-06T21:49:46.760113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install miceforest","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:43:41.443092Z","iopub.execute_input":"2022-05-06T20:43:41.445117Z","iopub.status.idle":"2022-05-06T20:43:50.595767Z","shell.execute_reply.started":"2022-05-06T20:43:41.445078Z","shell.execute_reply":"2022-05-06T20:43:50.594841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Imputation and Encoding","metadata":{}},{"cell_type":"code","source":"x_train = train_x_pivot.drop(['id', 'time'], axis=1).copy()\nx_test = test_x_pivot.drop(['id', 'time'], axis=1).copy()\nfrom sklearn.preprocessing import RobustScaler, LabelEncoder\nimport miceforest as mf\nX_train = mf.ampute_data(x_train)\nX_test = mf.ampute_data(x_test)\nRS = RobustScaler()\nX_train = RS.fit_transform(X_train)\nX_test = RS.transform(X_test)\n# Reshape \nX_train = X_train.reshape(-1, 10, X_train.shape[-1])\nX_test = X_test.reshape(-1, 10, X_test.shape[-1])","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:49:46.762154Z","iopub.execute_input":"2022-05-06T21:49:46.762892Z","iopub.status.idle":"2022-05-06T21:49:50.100892Z","shell.execute_reply.started":"2022-05-06T21:49:46.762849Z","shell.execute_reply":"2022-05-06T21:49:50.099953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = np.where(np.isnan(X_train), np.nanmedian(X_train, axis=0), X_train)\nX_test = np.where(np.isnan(X_test), np.nanmedian(X_test, axis=0), X_test)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:49:50.102777Z","iopub.execute_input":"2022-05-06T21:49:50.103034Z","iopub.status.idle":"2022-05-06T21:49:51.790119Z","shell.execute_reply.started":"2022-05-06T21:49:50.103005Z","shell.execute_reply":"2022-05-06T21:49:51.789351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:43:55.534186Z","iopub.execute_input":"2022-05-06T20:43:55.534433Z","iopub.status.idle":"2022-05-06T20:43:55.538094Z","shell.execute_reply.started":"2022-05-06T20:43:55.534399Z","shell.execute_reply":"2022-05-06T20:43:55.537171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\nencoder.fit(train_y)\nencoded_y = encoder.transform(train_y)\n# convert integers to dummy variables (i.e. one hot encoded)\ndummy_y = to_categorical(encoded_y)\n# \nprint(encoded_y.shape)\nprint(dummy_y.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:49:51.791277Z","iopub.execute_input":"2022-05-06T21:49:51.79153Z","iopub.status.idle":"2022-05-06T21:49:51.831126Z","shell.execute_reply.started":"2022-05-06T21:49:51.791495Z","shell.execute_reply":"2022-05-06T21:49:51.830336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using GPU\nimport os\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n\n# USE MULTIPLE GPUS\nif os.environ[\"CUDA_VISIBLE_DEVICES\"].count(',') == 0:\n    gpu_strategy = tf.distribute.get_strategy()\n    print('single strategy')\nelse:\n    gpu_strategy = tf.distribute.MirroredStrategy()\n    print('multiple strategy')","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:43:55.583861Z","iopub.execute_input":"2022-05-06T20:43:55.5841Z","iopub.status.idle":"2022-05-06T20:43:55.590001Z","shell.execute_reply.started":"2022-05-06T20:43:55.584068Z","shell.execute_reply":"2022-05-06T20:43:55.589133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformer Model\n> https://keras.io/examples/timeseries/timeseries_transformer_classification/","metadata":{}},{"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.python.keras import backend as K\n\n# adjust values to your needs\nconfig = tf.compat.v1.ConfigProto( device_count = {'GPU': 1 , 'CPU': 8} )\nsess = tf.compat.v1.Session(config=config) \nK.set_session(sess)\ndef transformer_encoder(inputs, head_size, num_heads, ff_dim, dropout=0):\n    # Normalization and Attention\n    x = layers.LayerNormalization(epsilon=1e-6)(inputs)\n    x = layers.MultiHeadAttention(\n        key_dim=head_size, num_heads=num_heads, dropout=dropout\n    )(x, x)\n    x = layers.Dropout(dropout)(x)\n    res = x + inputs\n\n    # Feed Forward Part\n    x = layers.LayerNormalization(epsilon=1e-6)(res)\n    x = layers.Conv1D(filters=ff_dim, kernel_size=1, activation=\"relu\")(x)\n    x = layers.Dropout(dropout)(x)\n    x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)\n    return x + res\ndef build_model(\n    input_shape,\n    head_size,\n    num_heads,\n    ff_dim,\n    num_transformer_blocks,\n    mlp_units,\n    dropout=0,\n    mlp_dropout=0\n):\n    inputs = keras.Input(shape=input_shape)\n    x = inputs\n    for _ in range(num_transformer_blocks):\n        x = transformer_encoder(x, head_size, num_heads, ff_dim, dropout)\n\n    x = layers.GlobalAveragePooling1D(data_format=\"channels_first\")(x)\n    for dim in mlp_units:\n        x = layers.Dense(dim, activation=\"relu\")(x)\n        x = layers.Dropout(mlp_dropout)(x)\n    outputs = layers.Dense(3, activation=\"softmax\")(x)\n    return keras.Model(inputs, outputs)\ninput_shape = X_train.shape[1:]\n\nmodelt = build_model(\n    input_shape,\n    head_size=256,\n    num_heads=4,\n    ff_dim=4,\n    num_transformer_blocks=4,\n    mlp_units=[128],\n    mlp_dropout=0.4,\n    dropout=0.25\n)\n\nmodelt.compile(\n    loss=\"sparse_categorical_crossentropy\",\n    optimizer=keras.optimizers.Adam(learning_rate=1e-4),\n    metrics=[\"sparse_categorical_accuracy\"],\n)\nmodelt.summary()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:43:55.643491Z","iopub.execute_input":"2022-05-06T20:43:55.643915Z","iopub.status.idle":"2022-05-06T20:43:55.990516Z","shell.execute_reply.started":"2022-05-06T20:43:55.643863Z","shell.execute_reply":"2022-05-06T20:43:55.989835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks = [keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)]\n\nmodelt.fit(\n    X_train,\n    encoded_y,\n    validation_split=0.2,\n    epochs=200,\n    batch_size=64,\n    callbacks=callbacks\n)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:43:55.991755Z","iopub.execute_input":"2022-05-06T20:43:55.991989Z","iopub.status.idle":"2022-05-06T20:54:10.507222Z","shell.execute_reply.started":"2022-05-06T20:43:55.991954Z","shell.execute_reply":"2022-05-06T20:54:10.506502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = []\ntest_preds.append(modelt.predict(X_test).squeeze())","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:54:10.508632Z","iopub.execute_input":"2022-05-06T20:54:10.508894Z","iopub.status.idle":"2022-05-06T20:54:28.202304Z","shell.execute_reply.started":"2022-05-06T20:54:10.508859Z","shell.execute_reply":"2022-05-06T20:54:28.20154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = sum(test_preds)\nsub = pd.DataFrame(predictions,columns=['away', 'draw', 'home'])\n#do not forget the id column\nsub['id'] = test[['id']]","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:54:28.203837Z","iopub.execute_input":"2022-05-06T20:54:28.204086Z","iopub.status.idle":"2022-05-06T20:54:28.212517Z","shell.execute_reply.started":"2022-05-06T20:54:28.204051Z","shell.execute_reply":"2022-05-06T20:54:28.211704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# modelt.predict(X_train).squeeze()\n# print(accuracy_score(encoded_y,[np.argmax(x) for x in modelt.predict(X_train).squeeze()]))\n# confusion_matrix(encoded_y,[np.argmax(x) for x in modelt.predict(X_train).squeeze()])","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:54:28.214029Z","iopub.execute_input":"2022-05-06T20:54:28.214291Z","iopub.status.idle":"2022-05-06T20:55:10.303249Z","shell.execute_reply.started":"2022-05-06T20:54:28.214256Z","shell.execute_reply":"2022-05-06T20:55:10.302293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM Model\n> https://www.analyticsvidhya.com/blog/2021/08/complete-guide-on-how-to-use-lightgbm-in-python/#:~:text=The%20main%20features%20of%20the,Parallel%20Learning%20support.","metadata":{}},{"cell_type":"code","source":"# LGB MODEL\n\n# Feature engineering again!!! Group by id, stats over time\ndef agg_features(df):\n    # vol_cols = ['log_return1_realized_volatility', 'log_return2_realized_volatility']\n    # Group by match id\n    df_agg = df.groupby(['id'])[feature_groups].agg(['mean', 'median', 'std', 'max', 'min', 'sum',]).reset_index()\n    # Rename columns joining suffix\n    df_agg.columns = ['_'.join(col) for col in df_agg.columns]\n    return df_agg\n    \ntrain_x_agg = agg_features(train_x_pivot)\ntest_x_agg = agg_features(test_x_pivot)\n\n# Taking Latest 3 matches data to predict outcome\n\ntrain_x_last = train_x_pivot.loc[train_x_pivot['time']==10]\ntest_x_last = test_x_pivot.loc[test_x_pivot['time']==10]\ntrain_x1_last = train_x_pivot.loc[train_x_pivot['time']==9]\ntest_x1_last = test_x_pivot.loc[test_x_pivot['time']==9]\ntrain_x2_last = train_x_pivot.loc[train_x_pivot['time']==8]\ntest_x2_last = test_x_pivot.loc[test_x_pivot['time']==8]\n\n\ntrain_x_last=(train_x_last.merge(train_x1_last,on='id',suffixes=('_10','_9'))).merge(train_x2_last,left_on='id',right_on='id',suffixes=('','_8'))\ntest_x_last=(test_x_last.merge(test_x1_last,on='id',suffixes=('_10','_9'))).merge(test_x2_last,left_on='id',right_on='id',suffixes=('','_8'))\n\nx_train = pd.merge(train_x_last, train_x_agg, left_on=\"id\", right_on=\"id_\").drop(['time_10','time_9','id','time'], axis = 1).copy()\nx_test = pd.merge(test_x_last, test_x_agg, left_on=\"id\", right_on=\"id_\").drop(['time_10','time_9','id','time'], axis = 1).copy()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:53:52.111037Z","iopub.execute_input":"2022-05-06T21:53:52.111322Z","iopub.status.idle":"2022-05-06T21:53:57.383911Z","shell.execute_reply.started":"2022-05-06T21:53:52.11129Z","shell.execute_reply":"2022-05-06T21:53:57.383025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target2int = {'away': 0, 'draw': 1, 'home': 2}\n# encode target\ndummy_y = train_y.map(target2int)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:54:07.304235Z","iopub.execute_input":"2022-05-06T21:54:07.304904Z","iopub.status.idle":"2022-05-06T21:54:07.320451Z","shell.execute_reply.started":"2022-05-06T21:54:07.304862Z","shell.execute_reply":"2022-05-06T21:54:07.31969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nnum_boost_round = 1000\nSEED = 123\nseed = SEED\nN_SPLITS = 3\nFIRST_OOF_ONLY= False\nlgbm_params = {\n    \"objective\":\"multiclass\" #\"binary\"\n    , \"boosting_type\":\"gbdt\"\n    , 'num_class':3\n    , 'metric': \"multi_logloss\" #''binary_logloss\n    ,        'learning_rate': 0.05,        \n            'lambda_l1': 2,\n            'lambda_l2': 7,\n            'num_leaves': 400,\n            'min_sum_hessian_in_leaf': 20,\n            'feature_fraction': 0.5,\n            'feature_fraction_bynode': 0.5,\n            'bagging_fraction': 0.5,\n            'bagging_freq': 42,\n            'min_data_in_leaf': 700,\n            'max_depth': 5,\n            'seed': seed,\n            'feature_fraction_seed': seed,\n            'bagging_seed': seed,\n            'drop_seed': seed,\n            'data_random_seed': seed,\n            'verbosity': -1\n        }\n\ntest_preds = []\n\nkf = kf = StratifiedKFold(n_splits=N_SPLITS, random_state=SEED, shuffle=True) \n\nfor fold, (train_idx, test_idx) in enumerate(kf.split(x_train, dummy_y)):\n    print('-'*15, '>', f'Fold {fold+1}/{N_SPLITS}', '<', '-'*15)\n    X_train, X_valid = x_train.iloc[train_idx], x_train.iloc[test_idx]\n    Y_train, Y_valid = dummy_y.iloc[train_idx], dummy_y.iloc[test_idx]\n    # \n    train_dataset = lgb.Dataset(X_train, Y_train) \n    val_dataset = lgb.Dataset(X_valid, Y_valid)\n    model = lgb.train(params = lgbm_params, \n                        train_set = train_dataset, \n                        valid_sets = [train_dataset, val_dataset], \n                        num_boost_round = num_boost_round, \n                        callbacks=[lgb.early_stopping(stopping_rounds=50)],\n                        verbose_eval = 50)\n    # Model validation    \n    y_true = Y_valid.squeeze()\n    y_pred = model.predict(X_valid).squeeze()\n    score1 = log_loss(y_true, y_pred)\n    print(f\"Fold-{fold+1} | OOF LogLoss Score: {score1}\")\n    \n    # Predictions\n    test_preds.append(model.predict(x_test).squeeze())\n    #lgb.plot_importance(model,max_num_features=20)\n    if FIRST_OOF_ONLY: break","metadata":{"execution":{"iopub.status.busy":"2022-05-06T20:55:15.43884Z","iopub.execute_input":"2022-05-06T20:55:15.439127Z","iopub.status.idle":"2022-05-06T21:03:08.771733Z","shell.execute_reply.started":"2022-05-06T20:55:15.439092Z","shell.execute_reply":"2022-05-06T21:03:08.771091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n#validation_accuracy_score\nprint(accuracy_score(y_true,[np.argmax(x) for x in model.predict(X_valid).squeeze()]))\nconfusion_matrix(y_true,[np.argmax(x) for x in model.predict(X_valid).squeeze()])","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:03:08.773071Z","iopub.execute_input":"2022-05-06T21:03:08.773554Z","iopub.status.idle":"2022-05-06T21:03:09.473183Z","shell.execute_reply.started":"2022-05-06T21:03:08.773515Z","shell.execute_reply":"2022-05-06T21:03:09.472513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Draw matches are not being predicted correctly. This maybe due to class imbalance !\n","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:07:32.579697Z","iopub.execute_input":"2022-05-06T23:07:32.579975Z","iopub.status.idle":"2022-05-06T23:07:32.584183Z","shell.execute_reply.started":"2022-05-06T23:07:32.579945Z","shell.execute_reply":"2022-05-06T23:07:32.583493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = np.where(np.isnan(x_train), np.nanmedian(x_train, axis=0), x_train)\nX_test = np.where(np.isnan(x_test), np.nanmedian(x_test, axis=0), x_test)\nX_train=pd.DataFrame(X_train,columns=x_train.columns)\nX_test=pd.DataFrame(X_test,columns=x_test.columns)\nX_train=X_train.drop((X_train).columns[pd.DataFrame(X_train).isna().sum().values>0],axis=1)\nX_test=X_test.drop((X_test).columns[pd.DataFrame(X_test).isna().sum().values>0],axis=1)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:57:37.015847Z","iopub.execute_input":"2022-05-06T21:57:37.016314Z","iopub.status.idle":"2022-05-06T21:57:37.225298Z","shell.execute_reply.started":"2022-05-06T21:57:37.016276Z","shell.execute_reply":"2022-05-06T21:57:37.224547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SMOTE Upsampling","metadata":{}},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\noversample = SMOTE()\nX_train, Y_train = oversample.fit_resample(X_train, dummy_y)\nY_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:57:45.174445Z","iopub.execute_input":"2022-05-06T21:57:45.174727Z","iopub.status.idle":"2022-05-06T21:58:33.689171Z","shell.execute_reply.started":"2022-05-06T21:57:45.174696Z","shell.execute_reply":"2022-05-06T21:58:33.688468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_master=X_train.copy()\nY_train_master=Y_train.copy()\nfrom sklearn.metrics import accuracy_score\nnum_boost_round = 1000\nSEED = 123\nseed = SEED\nN_SPLITS = 3 # 5\nFIRST_OOF_ONLY= False\nlgbm_params = {\n    \"objective\":\"multiclass\" #\"binary\"\n    , \"boosting_type\":\"gbdt\"\n    , 'num_class':3\n    , 'metric': \"multi_logloss\" #''binary_logloss\n    ,        'learning_rate': 0.05,        \n            'lambda_l1': 2,\n            'lambda_l2': 7,\n            'num_leaves': 400,\n            'min_sum_hessian_in_leaf': 20,\n            'feature_fraction': 0.5,\n            'feature_fraction_bynode': 0.5,\n            'bagging_fraction': 0.5,\n            'bagging_freq': 42,\n            'min_data_in_leaf': 700,\n            'max_depth': 5,\n            'seed': seed,\n            'feature_fraction_seed': seed,\n            'bagging_seed': seed,\n            'drop_seed': seed,\n            'data_random_seed': seed,\n            'verbosity': -1\n        }\n\ntest_preds = []\n\nkf = kf = StratifiedKFold(n_splits=N_SPLITS, random_state=SEED, shuffle=True) # KFold(n_splits=N_SPLITS, random_state=SEED, shuffle=True)\n\nfor fold, (train_idx, test_idx) in enumerate(kf.split(X_train_master, Y_train_master)):\n    print('-'*15, '>', f'Fold {fold+1}/{N_SPLITS}', '<', '-'*15)\n    X_train, X_valid = X_train_master.iloc[train_idx], X_train_master.iloc[test_idx]\n    Y_train, Y_valid = Y_train_master.iloc[train_idx], Y_train_master.iloc[test_idx]\n    # \n    train_dataset = lgb.Dataset(X_train, Y_train) #, categorical_feature = ['is_cup'])\n    val_dataset = lgb.Dataset(X_valid, Y_valid)# , categorical_feature = ['is_cup'])\n    model = lgb.train(params = lgbm_params, \n                        train_set = train_dataset, \n                        valid_sets = [train_dataset, val_dataset], \n                        num_boost_round = num_boost_round, \n                        callbacks=[lgb.early_stopping(stopping_rounds=50)],\n                        verbose_eval = 50)\n    # Model validation    \n    y_true = Y_valid.squeeze()\n    y_pred = model.predict(X_valid).squeeze()\n    score1 = log_loss(y_true, y_pred)\n    print(f\"Fold-{fold+1} | OOF LogLoss Score: {score1}\")\n    # print(roc_auc_score(y_true, y_pred))\n    # Predictions\n    test_preds.append(model.predict(X_test).squeeze())\n    #lgb.plot_importance(model,max_num_features=20)\n    if FIRST_OOF_ONLY: break","metadata":{"execution":{"iopub.status.busy":"2022-05-06T21:59:21.691022Z","iopub.execute_input":"2022-05-06T21:59:21.691277Z","iopub.status.idle":"2022-05-06T22:14:03.681621Z","shell.execute_reply.started":"2022-05-06T21:59:21.691247Z","shell.execute_reply":"2022-05-06T22:14:03.681016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(accuracy_score(y_true,[np.argmax(x) for x in model.predict(X_valid).squeeze()]))\nconfusion_matrix(y_true,[np.argmax(x) for x in model.predict(X_valid).squeeze()])","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:14:03.685326Z","iopub.execute_input":"2022-05-06T22:14:03.687069Z","iopub.status.idle":"2022-05-06T22:14:20.29718Z","shell.execute_reply.started":"2022-05-06T22:14:03.687032Z","shell.execute_reply":"2022-05-06T22:14:20.296492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = sum(test_preds)/N_SPLITS \n\n# away, draw, home\nsub_2 = pd.DataFrame(predictions,columns=['away', 'draw', 'home'])\n\n#do not forget the id column\nsub_2['id'] = test[['id']]","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:17:28.937374Z","iopub.execute_input":"2022-05-06T22:17:28.937665Z","iopub.status.idle":"2022-05-06T22:17:28.946972Z","shell.execute_reply.started":"2022-05-06T22:17:28.93762Z","shell.execute_reply":"2022-05-06T22:17:28.946221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combining Predictions of both models","metadata":{}},{"cell_type":"code","source":"def draw(x):\n    if x[1]>x[0]:\n        if x[1]>x[2]:\n            return 1\n    return -1\n    \nsub_2['draw_check']=sub_2.apply(draw,axis=1)\n            ","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:17:33.519132Z","iopub.execute_input":"2022-05-06T22:17:33.519911Z","iopub.status.idle":"2022-05-06T22:17:34.552861Z","shell.execute_reply.started":"2022-05-06T22:17:33.519859Z","shell.execute_reply":"2022-05-06T22:17:34.552078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final=sub.merge(sub_2,on='id',suffixes=('_1','_2'))","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:37:03.770592Z","iopub.execute_input":"2022-05-06T22:37:03.771262Z","iopub.status.idle":"2022-05-06T22:37:03.793733Z","shell.execute_reply.started":"2022-05-06T22:37:03.771225Z","shell.execute_reply":"2022-05-06T22:37:03.793009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For draw predictions by LGBM, assigning its more weightage to LGBM.\n# For Away and Home, taking mean.\ndef combine_models_draw(x):\n    if x[-1]==1:\n        return  [(x[0]*0.3+x[4]*0.7),(x[1]*0.3+x[5]*0.7),(x[2]*0.3+x[6]*0.7)]\n    else:\n        return [(x[0]*0.5+x[4]*0.5),(x[1]*0.5+x[5]*0.5),(x[2]*0.5+x[6]*0.5)]\n    ","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:37:05.144836Z","iopub.execute_input":"2022-05-06T22:37:05.145228Z","iopub.status.idle":"2022-05-06T22:37:05.155511Z","shell.execute_reply.started":"2022-05-06T22:37:05.145186Z","shell.execute_reply":"2022-05-06T22:37:05.154541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final['result']=final.apply(combine_models_draw,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:37:05.907806Z","iopub.execute_input":"2022-05-06T22:37:05.909853Z","iopub.status.idle":"2022-05-06T22:37:07.874604Z","shell.execute_reply.started":"2022-05-06T22:37:05.909786Z","shell.execute_reply":"2022-05-06T22:37:07.87385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final=final[['id','result']]\n#away-draw-home","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:37:09.492479Z","iopub.execute_input":"2022-05-06T22:37:09.493336Z","iopub.status.idle":"2022-05-06T22:37:09.501557Z","shell.execute_reply.started":"2022-05-06T22:37:09.493281Z","shell.execute_reply":"2022-05-06T22:37:09.500679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission=pd.concat([final[['id']],pd.DataFrame(final['result'].to_list(), columns = ['away', 'draw', 'home'])],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:37:11.508422Z","iopub.execute_input":"2022-05-06T22:37:11.508904Z","iopub.status.idle":"2022-05-06T22:37:11.577298Z","shell.execute_reply.started":"2022-05-06T22:37:11.508866Z","shell.execute_reply":"2022-05-06T22:37:11.576595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:38:06.806287Z","iopub.execute_input":"2022-05-06T22:38:06.806958Z","iopub.status.idle":"2022-05-06T22:38:07.301515Z","shell.execute_reply.started":"2022-05-06T22:38:06.806919Z","shell.execute_reply":"2022-05-06T22:38:07.300702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:38:00.823624Z","iopub.execute_input":"2022-05-06T22:38:00.824117Z","iopub.status.idle":"2022-05-06T22:38:00.837096Z","shell.execute_reply.started":"2022-05-06T22:38:00.824078Z","shell.execute_reply":"2022-05-06T22:38:00.836273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}