{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime as dt\n#\nfrom tensorflow import keras\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import RobustScaler, LabelEncoder\nfrom sklearn.metrics import accuracy_score, log_loss\n#\nimport tensorflow as tf\nfrom tensorflow.keras.utils import plot_model, to_categorical\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, LearningRateScheduler\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\nfrom tensorflow.keras import layers\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-04T14:22:05.910995Z","iopub.execute_input":"2022-05-04T14:22:05.911728Z","iopub.status.idle":"2022-05-04T14:22:13.487506Z","shell.execute_reply.started":"2022-05-04T14:22:05.911587Z","shell.execute_reply":"2022-05-04T14:22:13.486557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#loading the data\ntrain = pd.read_csv('/kaggle/input/football-match-probability-prediction/train.csv')\ntest = pd.read_csv('/kaggle/input/football-match-probability-prediction/test.csv')\nsubmission = pd.read_csv('/kaggle/input/football-match-probability-prediction/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:13.489302Z","iopub.execute_input":"2022-05-04T14:22:13.489950Z","iopub.status.idle":"2022-05-04T14:22:25.795356Z","shell.execute_reply.started":"2022-05-04T14:22:13.489909Z","shell.execute_reply":"2022-05-04T14:22:25.794633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set seed\nnp.random.seed(123)\ntf.random.set_seed(123)\n\n#Some parameters\nMASK = -666 # fill NA with -666 (the number of the beast)\nT_HIST = 10 # time history, last 10 games\nCLASS = 3 #number of classes (home, draw, away)\n\nDEBUG = False\n# Run on a small sample of the data\nif DEBUG:\n    train = train[:10000]","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:36.712433Z","iopub.execute_input":"2022-05-04T14:22:36.712739Z","iopub.status.idle":"2022-05-04T14:22:36.718661Z","shell.execute_reply.started":"2022-05-04T14:22:36.712711Z","shell.execute_reply":"2022-05-04T14:22:36.717643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exclude matches with no history at date 1 - full of NA (1159 rows excluded)\ntrain.dropna(subset=['home_team_history_match_date_1'], inplace = True)\n# exclude leagues with less than 15 matches: This change has no effect on the final log loss\nif False:\n    train['league_id_count'] = train.groupby('league_id')['id'].transform('count')\n    train = train.loc[train['league_id_count'] > 15]","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:37.830254Z","iopub.execute_input":"2022-05-04T14:22:37.830606Z","iopub.status.idle":"2022-05-04T14:22:38.021137Z","shell.execute_reply.started":"2022-05-04T14:22:37.830573Z","shell.execute_reply":"2022-05-04T14:22:38.019979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train: {train.shape} \\n Submission: {submission.shape}\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:38.584288Z","iopub.execute_input":"2022-05-04T14:22:38.584594Z","iopub.status.idle":"2022-05-04T14:22:38.629053Z","shell.execute_reply.started":"2022-05-04T14:22:38.584566Z","shell.execute_reply":"2022-05-04T14:22:38.628107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Global = pd.read_csv(\"../input/stacking-network-football/Global.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:40.690979Z","iopub.execute_input":"2022-05-04T14:22:40.691641Z","iopub.status.idle":"2022-05-04T14:22:40.695892Z","shell.execute_reply.started":"2022-05-04T14:22:40.691604Z","shell.execute_reply":"2022-05-04T14:22:40.694977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Names_models = [\"Logistic_regression\",\"Stochastic_Gradient_descent\",\"Naive_Bayes_Complementary\",\n               #\"Naive_Bayes_Multinomial\",\"Bagging_Complement_NB\",\"Bagging_LR\",\"Random_forest\",\n               #\"AdaBoost\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:40.860747Z","iopub.execute_input":"2022-05-04T14:22:40.861604Z","iopub.status.idle":"2022-05-04T14:22:40.867236Z","shell.execute_reply.started":"2022-05-04T14:22:40.861536Z","shell.execute_reply":"2022-05-04T14:22:40.865951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#New_features = []\n#for i in range(len(Names_models)):\n    #for j in [f'away_{Names_models[i]}', f'draw_{Names_models[i]}', f'home_{Names_models[i]}']:\n        #New_features.append(j)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:41.413462Z","iopub.execute_input":"2022-05-04T14:22:41.414237Z","iopub.status.idle":"2022-05-04T14:22:41.418430Z","shell.execute_reply.started":"2022-05-04T14:22:41.414188Z","shell.execute_reply":"2022-05-04T14:22:41.417499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#len(New_features)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:41.999389Z","iopub.execute_input":"2022-05-04T14:22:42.000544Z","iopub.status.idle":"2022-05-04T14:22:42.004034Z","shell.execute_reply.started":"2022-05-04T14:22:42.000500Z","shell.execute_reply":"2022-05-04T14:22:42.003371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Global[New_features].head()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:42.520731Z","iopub.execute_input":"2022-05-04T14:22:42.521343Z","iopub.status.idle":"2022-05-04T14:22:42.525122Z","shell.execute_reply.started":"2022-05-04T14:22:42.521305Z","shell.execute_reply":"2022-05-04T14:22:42.524344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train[New_features] = Global[New_features].loc[Global['target']!=3]\n#test[New_features] = Global[New_features].loc[Global['target']==3]","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:43.096615Z","iopub.execute_input":"2022-05-04T14:22:43.096891Z","iopub.status.idle":"2022-05-04T14:22:43.101024Z","shell.execute_reply.started":"2022-05-04T14:22:43.096862Z","shell.execute_reply":"2022-05-04T14:22:43.100175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Global_test = Global[New_features].loc[Global['target']==3]\n#Global_test.index = test.index\n#test[New_features] =Global_test[New_features]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test[New_features]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for cols \"date\", change to datatime \nfor col in train.filter(regex='date', axis=1).columns:\n    train[col] = pd.to_datetime(train[col])\n    test[col] = pd.to_datetime(test[col])\n\n# Some feature engineering\ndef add_features(df):\n    for i in range(1, 11): # range from 1 to 10\n        # Feat. difference of days\n        df[f'home_team_history_match_DIFF_day_{i}'] = (df['match_date'] - df[f'home_team_history_match_date_{i}']).dt.days\n        df[f'away_team_history_match_DIFF_days_{i}'] = (df['match_date'] - df[f'away_team_history_match_date_{i}']).dt.days\n    # Feat. difference of scored goals\n        df[f'home_team_history_DIFF_goal_{i}'] = df[f'home_team_history_goal_{i}'] - df[f'home_team_history_opponent_goal_{i}']\n        df[f'away_team_history_DIFF_goal_{i}'] = df[f'away_team_history_goal_{i}'] - df[f'away_team_history_opponent_goal_{i}']\n    # Feat dummy winner x loser\n        df[f'home_winner_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] > 0, 1., 0.) \n        df[f'home_loser_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] < 0, 1., 0.)\n        df[f'away_winner_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] > 0, 1., 0.)\n        df[f'away_loser_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] < 0, 1., 0.)\n    # Results: multiple nested where # away:0, draw:1, home:2\n        df[f'home_team_result_{i}'] = np.where(df[f'home_team_history_DIFF_goal_{i}'] > 0., 2.,\n                         (np.where(df[f'home_team_history_DIFF_goal_{i}'] == 0., 1,\n                                   np.where(df[f'home_team_history_DIFF_goal_{i}'].isna(), np.nan, 0))))\n        df[f'away_team_result_{i}'] = np.where(df[f'away_team_history_DIFF_goal_{i}'] > 0., 2.,\n                         (np.where(df[f'away_team_history_DIFF_goal_{i}'] == 0., 1.,\n                                   np.where(df[f'away_team_history_DIFF_goal_{i}'].isna(), np.nan, 0.))))\n    # Feat. difference of rating (\"modified\" ELO RATING)\n        df[f'home_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'home_team_history_opponent_rating_{i}']-df[f'home_team_history_rating_{i}'])/10))\n        df[f'away_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'away_team_history_opponent_rating_{i}']-df[f'away_team_history_rating_{i}'])/10))\n        df[f'home_away_team_history_ELO_rating_{i}'] = 1/(1+10**((df[f'away_team_history_rating_{i}']-df[f'home_team_history_rating_{i}'])/10))\n        # df[f'away_team_history_DIFF_rating_{i}'] =  - df[f'away_team_history_opponent_rating_{i}']\n    # Feat. same coach id\n        df[f'home_team_history_SAME_coaX_{i}'] = np.where(df['home_team_coach_id']==df[f'home_team_history_coach_{i}'],1,0)\n        df[f'away_team_history_SAME_coaX_{i}'] = np.where(df['away_team_coach_id']==df[f'away_team_history_coach_{i}'],1,0) \n    # Feat. same league id\n        df[f'home_team_history_SAME_leaG_{i}'] = np.where(df['league_id']==df[f'home_team_history_league_id_{i}'],1,0)\n        df[f'away_team_history_SAME_leaG_{i}'] = np.where(df['league_id']==df[f'away_team_history_league_id_{i}'],1,0) \n    # Fill NA with -666\n    # df.fillna(MASK, inplace = True)\n    return df\n\ntrain = add_features(train)\ntest = add_features(test)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:45.209089Z","iopub.execute_input":"2022-05-04T14:22:45.209801Z","iopub.status.idle":"2022-05-04T14:22:49.021703Z","shell.execute_reply.started":"2022-05-04T14:22:45.209750Z","shell.execute_reply":"2022-05-04T14:22:49.020607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save targets\n# train_id = train['id'].copy()\ntrain_y = train['target'].copy()\n#keep only some features\ntrain_x = train.drop(['target', 'home_team_name', 'away_team_name'], axis=1) #, inplace=True) # is_cup EXCLUDED\n# Exclude all date, league, coach columns\ntrain_x.drop(train.filter(regex='date').columns, axis=1, inplace = True)\ntrain_x.drop(train.filter(regex='league').columns, axis=1, inplace = True)\ntrain_x.drop(train.filter(regex='coach').columns, axis=1, inplace = True)\n#\n# Test set\n# test_id = test['id'].copy()\ntest_x = test.drop(['home_team_name', 'away_team_name'], axis=1)#, inplace=True) # is_cup EXCLUDED\n# Exclude all date, league, coach columns\ntest_x.drop(test.filter(regex='date').columns, axis=1, inplace = True)\ntest_x.drop(test.filter(regex='league').columns, axis=1, inplace = True)\ntest_x.drop(test.filter(regex='coach').columns, axis=1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:49.023342Z","iopub.execute_input":"2022-05-04T14:22:49.023753Z","iopub.status.idle":"2022-05-04T14:22:50.723183Z","shell.execute_reply.started":"2022-05-04T14:22:49.023720Z","shell.execute_reply":"2022-05-04T14:22:50.722421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Target, train and test shape\nprint(f\"Target: {train_y.shape} \\n Train shape: {train_x.shape} \\n Test: {test_x.shape}\")\nprint(f\"Column names: {list(train_x.columns)}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:50.724636Z","iopub.execute_input":"2022-05-04T14:22:50.724977Z","iopub.status.idle":"2022-05-04T14:22:50.732685Z","shell.execute_reply.started":"2022-05-04T14:22:50.724938Z","shell.execute_reply":"2022-05-04T14:22:50.731721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [ i for i in train_x.columns]","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:50.734604Z","iopub.execute_input":"2022-05-04T14:22:50.735087Z","iopub.status.idle":"2022-05-04T14:22:50.743333Z","shell.execute_reply.started":"2022-05-04T14:22:50.735056Z","shell.execute_reply":"2022-05-04T14:22:50.742511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(features)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:50.744795Z","iopub.execute_input":"2022-05-04T14:22:50.745122Z","iopub.status.idle":"2022-05-04T14:22:50.759551Z","shell.execute_reply.started":"2022-05-04T14:22:50.745081Z","shell.execute_reply":"2022-05-04T14:22:50.758455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:22:51.507617Z","iopub.execute_input":"2022-05-04T14:22:51.507930Z","iopub.status.idle":"2022-05-04T14:22:51.539546Z","shell.execute_reply.started":"2022-05-04T14:22:51.507901Z","shell.execute_reply":"2022-05-04T14:22:51.538890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:28:21.504668Z","iopub.execute_input":"2022-05-04T14:28:21.505154Z","iopub.status.idle":"2022-05-04T14:28:21.541005Z","shell.execute_reply.started":"2022-05-04T14:28:21.505121Z","shell.execute_reply":"2022-05-04T14:28:21.539722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_groups = [\"home_team_history_is_play_home\", \"home_team_history_is_cup\",\n    \"home_team_history_goal\", \"home_team_history_opponent_goal\",\n    \"home_team_history_rating\", \"home_team_history_opponent_rating\",  \n    \"away_team_history_is_play_home\", \"away_team_history_is_cup\",\n    \"away_team_history_goal\", \"away_team_history_opponent_goal\",\n    \"away_team_history_rating\", \"away_team_history_opponent_rating\",  \n    \"home_team_history_match_DIFF_day\", \"away_team_history_match_DIFF_days\",\n    \"home_team_history_DIFF_goal\",\"away_team_history_DIFF_goal\",\n    \"home_team_history_ELO_rating\",\"away_team_history_ELO_rating\",\n    \"home_away_team_history_ELO_rating\",\n    \"home_team_history_SAME_coaX\", \"away_team_history_SAME_coaX\",\n    \"home_team_history_SAME_leaG\", \"away_team_history_SAME_leaG\",\n    \"home_team_result\", \"away_team_result\",\n    \"home_winner\", \"home_loser\", \"away_winner\", \"away_loser\"]  #+ New_features   \n# Pivot dimension (id*features) x time_history\ntrain_x_pivot = pd.wide_to_long(train_x, stubnames=feature_groups, \n                i=['id','is_cup'], j='time', sep='_', suffix='\\d+')\ntest_x_pivot = pd.wide_to_long(test_x, stubnames=feature_groups, \n                i=['id','is_cup'], j='time', sep='_', suffix='\\d+')\n#\nprint(f\"Train pivot shape: {train_x_pivot.shape}\")  \nprint(f\"Test pivot shape: {test_x_pivot.shape}\") ","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:31:24.575466Z","iopub.execute_input":"2022-05-04T14:31:24.575804Z","iopub.status.idle":"2022-05-04T14:32:53.036055Z","shell.execute_reply.started":"2022-05-04T14:31:24.575769Z","shell.execute_reply":"2022-05-04T14:32:53.035092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create columns based on index\ntrain_x_pivot = train_x_pivot.reset_index()\ntest_x_pivot = test_x_pivot.reset_index()\n# Deal with the is_cup feature\n# There are NA in 'is_cup'\ntrain_x_pivot=train_x_pivot.fillna({'is_cup':False})\ntrain_x_pivot['is_cup'] = pd.get_dummies(train_x_pivot['is_cup'], drop_first=True)\n#\ntest_x_pivot=test_x_pivot.fillna({'is_cup':False})\ntest_x_pivot['is_cup']= pd.get_dummies(test_x_pivot['is_cup'], drop_first=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:32:53.038586Z","iopub.execute_input":"2022-05-04T14:32:53.039013Z","iopub.status.idle":"2022-05-04T14:32:53.758422Z","shell.execute_reply.started":"2022-05-04T14:32:53.038966Z","shell.execute_reply":"2022-05-04T14:32:53.757454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature engineering again!!! Group by id, stats over time\ndef add_features_II(df):\n    # goals\n    df['home_team_history_DIFF_goal_csum'] = df.groupby('id')['home_team_history_DIFF_goal'].cumsum()\n    df['away_team_history_DIFF_goal_csum'] = df.groupby('id')['away_team_history_DIFF_goal'].cumsum()\n    # df['home_team_hist_goal_csum'] = df.groupby('id')['home_team_history_goal'].cumsum()\n    # df['home_team_hist_opp_goal_csum'] = df.groupby('id')['home_team_history_opponent_goal'].cumsum()\n    # df['away_team_hist_goal_csum'] = df.groupby('id')['away_team_history_goal'].cumsum()\n    # df['away_team_hist_opp_goal_csum'] = df.groupby('id')['away_team_history_opponent_goal'].cumsum()\n    # rating\n    df['home_team_hist_rat_mean'] = df.groupby('id')['home_team_history_rating'].transform('mean')\n    df['away_team_hist_rat_mean'] = df.groupby('id')['away_team_history_rating'].transform('mean')\n    # df['away_team_hist_rat_mean'] = df.groupby('id')['away_team_history_rating'].mean()\n    df['home_away_rat_elo'] = 1/(1+10**((df['away_team_hist_rat_mean']-df['home_team_hist_rat_mean'])/10))\n    # Result (%)\n    df['home_team_result_mean'] = df.groupby('id')['home_team_result'].transform('mean')\n    df['away_team_result_mean'] = df.groupby('id')['away_team_result'].transform('mean')\n    # df['home_team_result_perc'] = df.groupby('id')['home_team_result'].cumsum()\n    # df['away_team_result_perc'] = df.groupby('id')['away_team_result'].cumsum()\n    # df['home_team_result_perc'] = df['home_team_result_perc']/(df['time']*2)\n    # df['away_team_result_perc'] = df['away_team_result_perc']/(df['time']*2)\n    # Lags result if INV = True\n    df['home_team_result_lag1'] = df.groupby('id')['home_team_result'].shift(-1)\n    df['away_team_result_lag1'] = df.groupby('id')['away_team_result'].shift(-1)\n    df['home_team_result_lag2'] = df.groupby('id')['home_team_result'].shift(-2)\n    df['away_team_result_lag2'] = df.groupby('id')['away_team_result'].shift(-2)\n    # dummies results\n    # df = pd.get_dummies(df['home_team_result'])\n    # df = pd.get_dummies(df['away_team_result'])\n    # df['one'] = 1\n    # df['count'] = df.groupby('id').isna()\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:32:53.759874Z","iopub.execute_input":"2022-05-04T14:32:53.760214Z","iopub.status.idle":"2022-05-04T14:32:53.771151Z","shell.execute_reply.started":"2022-05-04T14:32:53.760170Z","shell.execute_reply":"2022-05-04T14:32:53.770247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x_pivot = add_features_II(train_x_pivot)\ntest_x_pivot = add_features_II(test_x_pivot)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:32:53.772738Z","iopub.execute_input":"2022-05-04T14:32:53.773221Z","iopub.status.idle":"2022-05-04T14:32:54.857049Z","shell.execute_reply.started":"2022-05-04T14:32:53.773180Z","shell.execute_reply":"2022-05-04T14:32:54.855815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x_pivot","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:32:54.859371Z","iopub.execute_input":"2022-05-04T14:32:54.859706Z","iopub.status.idle":"2022-05-04T14:32:55.048290Z","shell.execute_reply.started":"2022-05-04T14:32:54.859663Z","shell.execute_reply":"2022-05-04T14:32:55.047229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INV = True\nif INV:\n    # Trying to keep the same id order\n    train_x_pivot.sort_values(by=['time'], inplace = True, ascending=False)\n    # Merge and drop columns\n    train_x_pivot = pd.merge(train_x['id'], train_x_pivot, on=\"id\").drop(['id', 'time'], axis = 1)\n    # Test\n    test_x_pivot.sort_values(by=['time'], inplace = True, ascending=False)\n    test_x_pivot = pd.merge(test_x['id'], test_x_pivot, on=\"id\").drop(['id', 'time'], axis = 1)\n    \n    # x_test_pivot = x_test_pivot.reset_index()\n    # x_test_pivot['time'] = (T_HIST + 1) - x_test_pivot['time']\n    # x_test_pivot.sort_values(by=['time'], inplace = True)\n    # x_test = pd.merge(test_id, x_test_pivot, on=\"id\").drop(['id', 'time'], axis = 1)\n    # x_test = x_test.to_numpy().reshape(-1, T_HIST, x_test.shape[-1])","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:35:25.970055Z","iopub.execute_input":"2022-05-04T14:35:25.970840Z","iopub.status.idle":"2022-05-04T14:35:31.959036Z","shell.execute_reply.started":"2022-05-04T14:35:25.970799Z","shell.execute_reply":"2022-05-04T14:35:31.957956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train_x_pivot.copy() #drop(['id', 'time'], axis=1)\nx_test = test_x_pivot.copy() #drop(['id', 'time'], axis=1)\n# Fill NA with median\nfill_median = True\nif fill_median:\n    x_train = np.where(np.isnan(x_train), np.nanmedian(x_train, axis=0), x_train)\n    x_test = np.where(np.isnan(x_test), np.nanmedian(x_test, axis=0), x_test)\n\n# Scale features using statistics that are robust to outliers\nRS = RobustScaler()\nx_train = RS.fit_transform(x_train)\nx_test = RS.transform(x_test)\n# Reshape \nx_train = x_train.reshape(-1, T_HIST, x_train.shape[-1])\nx_test = x_test.reshape(-1, T_HIST, x_test.shape[-1])\n\nif False:\n    # Fill NA with MASK\n    x_train = np.nan_to_num(x_train, nan=MASK)\n    x_test = np.nan_to_num(x_test, nan=MASK)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:35:31.960850Z","iopub.execute_input":"2022-05-04T14:35:31.961167Z","iopub.status.idle":"2022-05-04T14:35:36.900122Z","shell.execute_reply.started":"2022-05-04T14:35:31.961136Z","shell.execute_reply":"2022-05-04T14:35:36.898869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train array shape: {x_train.shape} \\nTest array shape: {x_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:35:36.901610Z","iopub.execute_input":"2022-05-04T14:35:36.901994Z","iopub.status.idle":"2022-05-04T14:35:36.909005Z","shell.execute_reply.started":"2022-05-04T14:35:36.901957Z","shell.execute_reply":"2022-05-04T14:35:36.907888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Deal with targets\n# encode class values as integers\nencoder = LabelEncoder()\nencoder.fit(train_y)\nencoded_y = encoder.transform(train_y)\n# convert integers to dummy variables (i.e. one hot encoded)\ndummy_y = to_categorical(encoded_y)\n# \nprint(encoded_y.shape)\nprint(dummy_y.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:36:11.137769Z","iopub.execute_input":"2022-05-04T14:36:11.138215Z","iopub.status.idle":"2022-05-04T14:36:11.194644Z","shell.execute_reply.started":"2022-05-04T14:36:11.138168Z","shell.execute_reply":"2022-05-04T14:36:11.193642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This is similar to the model of \"igorkf\" (Good, loss ~ 0.999)\ndef model_1():\n    x_input = layers.Input(shape=x_train.shape[1:])\n    #x = layers.Masking(mask_value=MASK, input_shape=(x_train.shape[1:]))(x_input)\n    #16\n    x = layers.Bidirectional(tf.compat.v1.keras.layers.LSTM(16, return_sequences=True))(x_input) #(x)\n    #x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.57)(x)  \n    #8\n    x = layers.Bidirectional(tf.compat.v1.keras.layers.LSTM(8, return_sequences=True))(x)\n    #x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.57)(x)\n    #x = layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNLSTM(4, return_sequences=True))(x)\n    #x = layers.Dropout(0.6)(x)\n    x = layers.Flatten()(x)\n    # output\n    output = layers.Dense(CLASS, activation='softmax')(x)\n\n    model = Model(inputs=[x_input],outputs=[output])\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-04T15:05:00.114647Z","iopub.execute_input":"2022-05-04T15:05:00.115752Z","iopub.status.idle":"2022-05-04T15:05:00.124606Z","shell.execute_reply.started":"2022-05-04T15:05:00.115693Z","shell.execute_reply":"2022-05-04T15:05:00.123429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Choose your model\nmodel = model_1()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-04T15:05:00.741738Z","iopub.execute_input":"2022-05-04T15:05:00.742051Z","iopub.status.idle":"2022-05-04T15:05:01.408672Z","shell.execute_reply.started":"2022-05-04T15:05:00.742021Z","shell.execute_reply":"2022-05-04T15:05:01.407575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(\n    model, \n    to_file='Football_Prob_Model.png', \n    show_shapes=True,\n    show_layer_names=True\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T15:05:04.468293Z","iopub.execute_input":"2022-05-04T15:05:04.468573Z","iopub.status.idle":"2022-05-04T15:05:04.720370Z","shell.execute_reply.started":"2022-05-04T15:05:04.468546Z","shell.execute_reply":"2022-05-04T15:05:04.719180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import os\n#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n\n# USE MULTIPLE GPUS\n#if os.environ[\"CUDA_VISIBLE_DEVICES\"].count(',') == 0:\n    #gpu_strategy = tf.distribute.get_strategy()\n    #print('single strategy')\n#else:\n    #gpu_strategy = tf.distribute.MirroredStrategy()\n    #('multiple strategy')","metadata":{"execution":{"iopub.status.busy":"2022-05-04T15:05:05.126534Z","iopub.execute_input":"2022-05-04T15:05:05.126859Z","iopub.status.idle":"2022-05-04T15:05:05.132517Z","shell.execute_reply.started":"2022-05-04T15:05:05.126826Z","shell.execute_reply":"2022-05-04T15:05:05.130868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# N_SPLITS of the traning set for validation using KFold\n# Parameters\nEPOCH = 200\nBATCH_SIZE = 64\nN_SPLITS = 15\nSEED = 123\nVERBOSE = 1\nPATIENCE = EPOCH // 10\n\ntest_preds = []\ntrain_preds = []\n# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n\n# instantiate a distribution strategy\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n\nwith tpu_strategy.scope():\n#with gpu_strategy.scope():\n    kf = KFold(n_splits=N_SPLITS, random_state=SEED,shuffle=True) # shuffle=Truecount = 0f\n    for fold, (train_idx, test_idx) in enumerate(kf.split(x_train, dummy_y)):\n        print('-'*15, '>', f'Fold {fold+1}/{N_SPLITS}', '<', '-'*15)\n        X_train, X_valid = x_train[train_idx], x_train[test_idx]\n        Y_train, Y_valid = dummy_y[train_idx], dummy_y[test_idx]\n######### Model: CHANGE HERE TOO ################\n        model = model_1()\n# It is a multi-class classification problem, categorical_crossentropy is used as the loss function.\n        #keras.optimizers.Adam\n        #optimizer = keras.optimizers.Adam(learning_rate=1e-8)#, momentum=0.9)\n        model.compile(optimizer=\"Adam\", loss=\"categorical_crossentropy\",\n                     metrics=[\"accuracy\"])\n        #\n        #lr_schedule = tf.keras.callbacks.LearningRateScheduler(\n                #lambda epoch: 1e-8 * 10**(EPOCH / 20))\n        \n        es = EarlyStopping(monitor='val_loss', patience=PATIENCE, verbose=0, mode='min',\n                           restore_best_weights=True)\n        lr = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=10, verbose=0)\n        #\n        history = model.fit(X_train, Y_train, \n                       validation_data=(X_valid, Y_valid), \n                       epochs=EPOCH,\n                       verbose=VERBOSE,\n                       batch_size=BATCH_SIZE,  \n                       callbacks=[lr, es])#,lr_schedule])\n        # Model validation    \n        y_true = Y_valid.squeeze()\n        y_pred = model.predict(X_valid, batch_size=BATCH_SIZE).squeeze()\n        score1 = log_loss(y_true, y_pred)\n        print(f\"Fold-{fold+1} | OOF LogLoss Score: {score1}\")\n        # Predictions\n        #if score1<1.0:\n        test_preds.append(model.predict(x_test).squeeze())\n        train_preds.append(model.predict(x_train).squeeze())","metadata":{"execution":{"iopub.status.busy":"2022-05-04T15:05:06.283482Z","iopub.execute_input":"2022-05-04T15:05:06.283790Z","iopub.status.idle":"2022-05-04T15:20:29.582322Z","shell.execute_reply.started":"2022-05-04T15:05:06.283757Z","shell.execute_reply":"2022-05-04T15:20:29.580883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = sum(test_preds)/N_SPLITS \n\n# away, draw, home\nsubmission = pd.DataFrame(predictions,columns=['away', 'draw', 'home'])\n\n# Round\nround_num = True\nif round_num:\n    submission = submission.round(2)\n    submission['draw'] = 1 - (submission['home'] + submission['away'])  \n    \n#do not forget the id column\nsubmission['id'] = test[['id']]\n\n#submit!\nsubmission[['id', 'home', 'away', 'draw']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-04T15:04:54.288005Z","iopub.status.idle":"2022-05-04T15:04:54.289086Z","shell.execute_reply.started":"2022-05-04T15:04:54.288816Z","shell.execute_reply":"2022-05-04T15:04:54.288845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[['id', 'home', 'away', 'draw']].head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = sum(train_preds)/N_SPLITS \n\n# away, draw, home\nsubmission_train = pd.DataFrame(predictions,columns=['away', 'draw', 'home'])\n\n# Round\nround_num = True\nif round_num:\n    submission_train =submission_train.round(2)\n    submission_train['draw'] = 1 - (submission_train['home'] + submission_train['away'])  \n    \n#do not forget the id column\nsubmission_train['id'] = test[['id']]\n\n#submit!\nsubmission_train[['id', 'home', 'away', 'draw']].to_csv('submission_train.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}