{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#
⚽️ Football Prediction | Pytorch 🔥 BI-LSTM
","metadata":{}},{"cell_type":"markdown","source":"# Import libraries and data","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-04T10:06:15.525859Z","iopub.execute_input":"2022-12-04T10:06:15.526287Z","iopub.status.idle":"2022-12-04T10:06:15.558273Z","shell.execute_reply.started":"2022-12-04T10:06:15.526203Z","shell.execute_reply":"2022-12-04T10:06:15.556780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport plotly.graph_objects as go\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:13:54.658326Z","iopub.execute_input":"2022-12-04T10:13:54.658802Z","iopub.status.idle":"2022-12-04T10:13:54.777533Z","shell.execute_reply.started":"2022-12-04T10:13:54.658768Z","shell.execute_reply":"2022-12-04T10:13:54.776011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def target_to_int(target):\n '''\n Mapping function for the labels to predict\n '''\n if target == 'home':\n return 0\n elif target == 'draw':\n return 1\n elif target == 'away':\n return 2","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:14:07.773490Z","iopub.execute_input":"2022-12-04T10:14:07.774006Z","iopub.status.idle":"2022-12-04T10:14:07.780793Z","shell.execute_reply.started":"2022-12-04T10:14:07.773967Z","shell.execute_reply":"2022-12-04T10:14:07.779797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing the data\nX_train = pd.read_csv('/kaggle/input/football-match-probability-prediction/train.csv').set_index('id')\nX_train.drop(['target'], axis=1, inplace=True)\ntarget_and_scores = pd.read_csv('/kaggle/input/football-match-probability-prediction/train_target_and_scores.csv').set_index('id')\ny_train = target_and_scores['target'].apply(lambda x: target_to_int(x))\nX_test = pd.read_csv('/kaggle/input/football-match-probability-prediction/test.csv').set_index('id')\n\n# We remove the last games above this variable (=10 if we want all the past games)\ndrop_last_games_above_i = 10","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:15:17.678443Z","iopub.execute_input":"2022-12-04T10:15:17.678947Z","iopub.status.idle":"2022-12-04T10:15:25.563027Z","shell.execute_reply.started":"2022-12-04T10:15:17.678911Z","shell.execute_reply":"2022-12-04T10:15:25.561879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop match with no history at date 1\nto_remove = X_train[X_train['home_team_history_match_date_1'].isna()].index\nX_train.drop(to_remove, axis=0, inplace=True)\ny_train.drop(to_remove, axis=0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:15:53.343392Z","iopub.execute_input":"2022-12-04T10:15:53.343813Z","iopub.status.idle":"2022-12-04T10:15:53.498077Z","shell.execute_reply.started":"2022-12-04T10:15:53.343779Z","shell.execute_reply":"2022-12-04T10:15:53.496285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"too_much_nans_goals_idxs = X_train[(X_train['home_team_history_goal_1'].isna()) & \n (X_train['home_team_history_goal_2'].isna()) &\n (X_train['home_team_history_goal_3'].isna()) &\n (X_train['home_team_history_goal_4'].isna()) &\n (X_train['home_team_history_goal_5'].isna()) &\n (X_train['home_team_history_goal_6'].isna()) &\n (X_train['home_team_history_goal_7'].isna()) &\n (X_train['home_team_history_goal_8'].isna()) &\n (X_train['home_team_history_goal_9'].isna()) &\n (X_train['home_team_history_goal_10'].isna()) &\n (X_train['away_team_history_goal_1'].isna()) & \n (X_train['away_team_history_goal_2'].isna()) &\n (X_train['away_team_history_goal_3'].isna()) &\n (X_train['away_team_history_goal_4'].isna()) &\n (X_train['away_team_history_goal_5'].isna()) &\n (X_train['away_team_history_goal_6'].isna()) &\n (X_train['away_team_history_goal_7'].isna()) &\n (X_train['away_team_history_goal_8'].isna()) &\n (X_train['away_team_history_goal_9'].isna()) &\n (X_train['away_team_history_goal_10'].isna())].index\n\nX_train.drop(too_much_nans_goals_idxs, axis=0, inplace=True)\ny_train.drop(too_much_nans_goals_idxs, axis=0, inplace=True)\n\ntoo_much_nans_goals_idxs = X_train[(X_train['home_team_history_rating_1'].isna()) & \n (X_train['home_team_history_rating_2'].isna()) &\n (X_train['home_team_history_rating_3'].isna()) &\n (X_train['home_team_history_rating_4'].isna()) &\n (X_train['home_team_history_rating_5'].isna()) &\n (X_train['home_team_history_rating_6'].isna()) &\n (X_train['home_team_history_rating_7'].isna()) &\n (X_train['home_team_history_rating_8'].isna()) &\n (X_train['home_team_history_rating_9'].isna()) &\n (X_train['home_team_history_rating_10'].isna()) &\n (X_train['away_team_history_rating_1'].isna()) & \n (X_train['away_team_history_rating_2'].isna()) &\n (X_train['away_team_history_rating_3'].isna()) &\n (X_train['away_team_history_rating_4'].isna()) &\n (X_train['away_team_history_rating_5'].isna()) &\n (X_train['away_team_history_rating_6'].isna()) &\n (X_train['away_team_history_rating_7'].isna()) &\n (X_train['away_team_history_rating_8'].isna()) &\n (X_train['away_team_history_rating_9'].isna()) &\n (X_train['away_team_history_rating_10'].isna())].index\n\nX_train.drop(too_much_nans_goals_idxs, axis=0, inplace=True)\ny_train.drop(too_much_nans_goals_idxs, axis=0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:16:05.532517Z","iopub.execute_input":"2022-12-04T10:16:05.532897Z","iopub.status.idle":"2022-12-04T10:16:05.674434Z","shell.execute_reply.started":"2022-12-04T10:16:05.532865Z","shell.execute_reply":"2022-12-04T10:16:05.673328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NaNs","metadata":{}},{"cell_type":"code","source":"all_nan = pd.concat([\n (X_train.isna().sum(axis=0)/X_train.shape[0]*100).rename('Train % NaN'),\n (X_test.isna().sum(axis=0)/X_test.shape[0]*100).rename('Test % NaN')\n ], \n axis=1) \\\n .sort_values(['Train % NaN', 'Test % NaN'], ascending=False)\n\nall_nan.style.background_gradient()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:16:33.867430Z","iopub.execute_input":"2022-12-04T10:16:33.867893Z","iopub.status.idle":"2022-12-04T10:16:34.166542Z","shell.execute_reply.started":"2022-12-04T10:16:33.867855Z","shell.execute_reply":"2022-12-04T10:16:34.165629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## NaNs filling logic: \n\nCoach related features have a lot of NaNs (at least 25% in test and 20% in train for current match, and it's increasing when we look at past matches)...\n\n- Descriptive columns:\n\n - **home_team_name** - The name of the Home the team. Hidden in test set, see this discussion\n - Not used (hidden in test set) \n \n - **away_team_name** - The name of the Away the team. Hidden in test set, see this discussion\n - Not used (hidden in test set) \n\n - **match_date** - The match date (UTC).\n - No NaNs\n\n - **league_name** - The league name.\n - Not used (we already have league_id) \n\n - **league_id** - The league id. Note that league names can be identical for two differents id.\n - No NaNs\n\n - **is_cup** - If the value is 1 the match is played for a cup compettion.\n - Train: Just 1 NaN -> league id 1101 corresponding to no cup -> Assume it's no cup\n - Test: No NaNs\n\n - **home_team_coach_id** - The id of the Home team coach
**away_team_coach_id** - The id of the Away team coach. (Same logic as home case)\n - Not used (too much NaNs)\n\n- Historical columns:\n\n - **home_team_history_match_date_i** - The date of the last i-th match played by Home team.
**away_team_history_match_date_i** - The date of the last i-th match played\n - We calculate the number of days between the past matches dates and the current one, then we take the median of the number of days, then we calculate the number of days between each match (by difference), and we add this number recursively (see example below)\n\n - **home_team_history_is_play_home_i** - If 1, the Home team played home.
**away_team_history_is_play_home_i** - If 1, the Away team played home. \n - If the team played at home in the previous game, then put False, otherwise True\n\n - **home_team_history_league_id_i** - The league name id by the Home team on its last i-th match.
**away_team_history_league_id_i** - The league name id played by the Away on its last i-th match.\n - Fill the current match league id \n\n - **home_team_history_is_cup_i** - If 1, the match was a cup competition.
**away_team_history_is_cup_i** - If 1, the match was a cup competition. \n - Assume it's no cup (but later have to do this with the league id associated with cup or no cups)\n\n - **home_team_history_goal_i** - The number of goals scored by the Home team on its last i-th match.
**away_team_history_goal_i** - The number of goals scored by the Away team on its last i-th match.
**home_team_history_opponent_goal_i** - The number of goals conceded by the Home team on its last i-th match.
**away_team_history_opponent_goal_i** - The number of goals conceded by the Away team on its last i-th match.\n - median grouped by league id (can't with team names because it's hidden in test set) \n \n - **home_team_history_rating_i** - The rating of the Home team on its last i-th match (pre match rating).
**away_team_history_rating_i** - The rating of the Away team on its last i-th match (pre match rating).
**home_team_history_opponent_rating_i** - The rating of the opponent team on Home team last i-th match (pre match rating).
**away_team_history_opponent_rating_i** - The rating of the opponent team on Away team last i-th match (pre match rating).\n - median grouped by league id (can't with team names because it's hidden in test set) \n \n - **home_team_history_coach_i** - The coach id of the Home team on its last i-th match.
**away_team_history_coach_i** - The coach id of the Away team on its last i-th match.\n - Not used (too much NaNs) \n\n---","metadata":{}},{"cell_type":"markdown","source":"## Historical Match Dates features\n\nLet's first compute the nb_days between past match dates and current date, and then calculate the median (better than the mean because for example if the 10-past match for a team is 365 days ago the mean will be really impacted, not the median)","metadata":{}},{"cell_type":"code","source":"df = X_train.copy()\nfor i in range(1, drop_last_games_above_i + 1):\n # Nb of days between match date and last matches\n df[f'home_team_history_nb_days_match_{i}'] = (pd.to_datetime(df['match_date']) - pd.to_datetime(df[f'home_team_history_match_date_{i}'])).apply(lambda x: x.days)\n df[f'away_team_history_nb_days_match_{i}'] = (pd.to_datetime(df['match_date']) - pd.to_datetime(df[f'away_team_history_match_date_{i}'])).apply(lambda x: x.days)\n df.drop([f'home_team_history_match_date_{i}', f'away_team_history_match_date_{i}'], axis=1, inplace=True)\n\n# Here the median nb days ago from current match date\ndf[[col for col in df.columns if 'nb_days' in col]].median()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:17:54.610364Z","iopub.execute_input":"2022-12-04T10:17:54.610755Z","iopub.status.idle":"2022-12-04T10:18:08.803100Z","shell.execute_reply.started":"2022-12-04T10:17:54.610722Z","shell.execute_reply":"2022-12-04T10:18:08.801590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see the median is the same for home and away team past matches, it strengthens the coherence.\n\nNow we are going to add the nb_days difference from past matches for home and away teams, for $X_{train}$ and $X_{test}$ (The first diff is 6 days ago), recursively for 1-10 past games","metadata":{}},{"cell_type":"code","source":"median_diff_nb_days_train = df[[col for col in df.columns if 'nb_days' in col]].median().drop_duplicates().diff().fillna(6.)\nmedian_diff_nb_days_train","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:19:17.203843Z","iopub.execute_input":"2022-12-04T10:19:17.204244Z","iopub.status.idle":"2022-12-04T10:19:17.261976Z","shell.execute_reply.started":"2022-12-04T10:19:17.204211Z","shell.execute_reply":"2022-12-04T10:19:17.260966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_nb_days_match(df, drop_last_games_above_i):\n # Computing nb_days between games and current date\n for i in range(1, drop_last_games_above_i + 1):\n df[f'home_team_history_nb_days_match_{i}'] = (pd.to_datetime(df['match_date']) - pd.to_datetime(df[f'home_team_history_match_date_{i}'])).apply(lambda x: x.days)\n df[f'away_team_history_nb_days_match_{i}'] = (pd.to_datetime(df['match_date']) - pd.to_datetime(df[f'away_team_history_match_date_{i}'])).apply(lambda x: x.days)\n df.drop([f'home_team_history_match_date_{i}', f'away_team_history_match_date_{i}'], axis=1, inplace=True)\n return df\n\ndef fill_na_nb_days_match(df, median_diff_nb_days, drop_last_games_above_i):\n # Call the nb_days computation function\n df = compute_nb_days_match(df, drop_last_games_above_i)\n\n # Fill NaNs for past game 1\n df.loc[df['home_team_history_nb_days_match_1'].isna(), 'home_team_history_nb_days_match_1'] = median_diff_nb_days['home_team_history_nb_days_match_1']\n df.loc[df['away_team_history_nb_days_match_1'].isna(), 'away_team_history_nb_days_match_1'] = median_diff_nb_days['home_team_history_nb_days_match_1']\n\n # Fill NaNs for other last games (2-10)\n for i in range(2, drop_last_games_above_i + 1):\n df.loc[df[f'home_team_history_nb_days_match_{i}'].isna(), \n f'home_team_history_nb_days_match_{i}'] = df.loc[df[f'home_team_history_nb_days_match_{i}'].isna(), \n f'home_team_history_nb_days_match_{i-1}'] \\\n + median_diff_nb_days[f'home_team_history_nb_days_match_{i}']\n df.loc[df[f'away_team_history_nb_days_match_{i}'].isna(), \n f'away_team_history_nb_days_match_{i}'] = df.loc[df[f'away_team_history_nb_days_match_{i}'].isna(), \n f'away_team_history_nb_days_match_{i-1}'] \\\n + median_diff_nb_days[f'home_team_history_nb_days_match_{i}']\n\n # We check that all the nb_days are increasing with time (columns are ordered 1-10)\n assert(all(df[[col for col in df.columns if 'home_team_history_nb_days' in col]].diff(axis=1) >= 0))\n assert(all(df[[col for col in df.columns if 'away_team_history_nb_days' in col]].diff(axis=1) >= 0))\n\n return df","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:19:19.061744Z","iopub.execute_input":"2022-12-04T10:19:19.062265Z","iopub.status.idle":"2022-12-04T10:19:19.075563Z","shell.execute_reply.started":"2022-12-04T10:19:19.062225Z","shell.execute_reply":"2022-12-04T10:19:19.074551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = fill_na_nb_days_match(X_train, median_diff_nb_days_train, drop_last_games_above_i)\nX_test = fill_na_nb_days_match(X_test, median_diff_nb_days_train, drop_last_games_above_i)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:19:19.770584Z","iopub.execute_input":"2022-12-04T10:19:19.771477Z","iopub.status.idle":"2022-12-04T10:19:42.928998Z","shell.execute_reply.started":"2022-12-04T10:19:19.771417Z","shell.execute_reply":"2022-12-04T10:19:42.927599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Historical league_id features","metadata":{}},{"cell_type":"code","source":"def fill_na_history_league_id(df, drop_last_games_above_i):\n df[[col for col in df.columns if 'history_league_id' in col]] = df[[col for col in df.columns if 'history_league_id' in col]].apply(lambda x: x.fillna(value=df['league_id']))\n return df","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:19:42.930844Z","iopub.execute_input":"2022-12-04T10:19:42.931251Z","iopub.status.idle":"2022-12-04T10:19:42.938473Z","shell.execute_reply.started":"2022-12-04T10:19:42.931211Z","shell.execute_reply":"2022-12-04T10:19:42.936786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = fill_na_history_league_id(X_train, drop_last_games_above_i)\nX_test = fill_na_history_league_id(X_test, drop_last_games_above_i)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:19:48.056718Z","iopub.execute_input":"2022-12-04T10:19:48.057099Z","iopub.status.idle":"2022-12-04T10:19:48.104901Z","shell.execute_reply.started":"2022-12-04T10:19:48.057068Z","shell.execute_reply":"2022-12-04T10:19:48.103533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Historical is_play_home features\n\nAssume first past game NaN are played home.\nThen alterntate recursively (just NaNs will be filled).","metadata":{}},{"cell_type":"code","source":"def fill_na_history_is_play_home(df, drop_last_games_above_i):\n # Fill NaNs for last game 1\n df.loc[df['home_team_history_is_play_home_1'].isna(), 'home_team_history_is_play_home_1'] = 0.\n df.loc[df['away_team_history_is_play_home_1'].isna(), 'away_team_history_is_play_home_1'] = 0.\n\n # Fill NaNs for past games (1->10)\n for i in range(2, drop_last_games_above_i + 1):\n df.loc[df[f'home_team_history_is_play_home_{i}'].isna(), f'home_team_history_is_play_home_{i}'] = (df.loc[df[f'home_team_history_is_play_home_{i}'].isna(), \n f'home_team_history_is_play_home_{i-1}'] - 1.).abs()\n df.loc[df[f'away_team_history_is_play_home_{i}'].isna(), f'away_team_history_is_play_home_{i}'] = (df.loc[df[f'away_team_history_is_play_home_{i}'].isna(), \n f'away_team_history_is_play_home_{i-1}'] - 1.).abs()\n return df","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:20:08.740439Z","iopub.execute_input":"2022-12-04T10:20:08.740976Z","iopub.status.idle":"2022-12-04T10:20:08.751678Z","shell.execute_reply.started":"2022-12-04T10:20:08.740936Z","shell.execute_reply":"2022-12-04T10:20:08.749416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = fill_na_history_is_play_home(X_train, drop_last_games_above_i)\nX_test = fill_na_history_is_play_home(X_test, drop_last_games_above_i)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:20:17.366999Z","iopub.execute_input":"2022-12-04T10:20:17.367491Z","iopub.status.idle":"2022-12-04T10:20:17.434410Z","shell.execute_reply.started":"2022-12-04T10:20:17.367438Z","shell.execute_reply":"2022-12-04T10:20:17.433665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### is_cup feature \n\nJust 1 NaN for the descriptive feature in train, we assume it's no cup. \n\nLet's compute alternatively for the historical features.","metadata":{}},{"cell_type":"code","source":"# Descriptive is_cup feature, just 1 in train set \nX_train.loc[X_train['is_cup'].isna(),'is_cup'] = 0","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:20:36.658752Z","iopub.execute_input":"2022-12-04T10:20:36.659125Z","iopub.status.idle":"2022-12-04T10:20:36.674129Z","shell.execute_reply.started":"2022-12-04T10:20:36.659095Z","shell.execute_reply":"2022-12-04T10:20:36.672339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_na_history_is_cup(df, drop_last_games_above_i):\n\n # Fill NaNs for past games (1-10)\n for i in range(1, drop_last_games_above_i + 1):\n df.loc[df[f'home_team_history_is_cup_{i}'].isna(), f'home_team_history_is_cup_{i}'] = 0.\n df.loc[df[f'away_team_history_is_cup_{i}'].isna(), f'away_team_history_is_cup_{i}'] = 0.\n\n return df","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:20:44.623341Z","iopub.execute_input":"2022-12-04T10:20:44.623780Z","iopub.status.idle":"2022-12-04T10:20:44.631966Z","shell.execute_reply.started":"2022-12-04T10:20:44.623746Z","shell.execute_reply":"2022-12-04T10:20:44.630517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = fill_na_history_is_cup(X_train, drop_last_games_above_i)\nX_test = fill_na_history_is_cup(X_test, drop_last_games_above_i)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:20:51.288767Z","iopub.execute_input":"2022-12-04T10:20:51.289203Z","iopub.status.idle":"2022-12-04T10:20:51.331231Z","shell.execute_reply.started":"2022-12-04T10:20:51.289162Z","shell.execute_reply":"2022-12-04T10:20:51.329760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Historical goals features\n\nFill NaNs with the corresponding league_id median number of goals\n\nWe are concatening X_train and X_test because some leagues in the test set are not in the train set. Normally, X_test should never be used !","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame()\nfor i in range(1, drop_last_games_above_i + 1):\n # Train set\n df = pd.concat([df,\n # Train set \n pd.DataFrame(X_train[[f'home_team_history_league_id_{i}', f'home_team_history_goal_{i}']].values),\n pd.DataFrame(X_train[[f'home_team_history_league_id_{i}', f'home_team_history_opponent_goal_{i}']].values),\n pd.DataFrame(X_train[[f'away_team_history_league_id_{i}', f'away_team_history_goal_{i}']].values),\n pd.DataFrame(X_train[[f'away_team_history_league_id_{i}', f'away_team_history_opponent_goal_{i}']].values),\n # Test set\n pd.DataFrame(X_test[[f'home_team_history_league_id_{i}', f'home_team_history_goal_{i}']].values),\n pd.DataFrame(X_test[[f'home_team_history_league_id_{i}', f'home_team_history_opponent_goal_{i}']].values),\n pd.DataFrame(X_test[[f'away_team_history_league_id_{i}', f'away_team_history_goal_{i}']].values),\n pd.DataFrame(X_test[[f'away_team_history_league_id_{i}', f'away_team_history_opponent_goal_{i}']].values)\n ], \n axis=0)\n \ndf.columns = ['league_id', 'nb_goals']\nmedian_leagues_id_goals = df.groupby('league_id').median()\nmedian_leagues_id_goals.fillna(value=median_leagues_id_goals.median(), inplace=True)\nmedian_leagues_id_goals","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:21:10.965498Z","iopub.execute_input":"2022-12-04T10:21:10.965949Z","iopub.status.idle":"2022-12-04T10:21:11.711084Z","shell.execute_reply.started":"2022-12-04T10:21:10.965912Z","shell.execute_reply":"2022-12-04T10:21:11.710137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_na_history_goals(df, median_leagues_id_goals, drop_last_games_above_i):\n\n # Fill NaNs for past games (1-10)\n for i in range(1, drop_last_games_above_i + 1):\n df.loc[df[f'home_team_history_goal_{i}'].isna(), f'home_team_history_goal_{i}'] = median_leagues_id_goals.loc[df.loc[df[f'home_team_history_goal_{i}'].isna(), f'home_team_history_league_id_{i}']].values\n df.loc[df[f'home_team_history_opponent_goal_{i}'].isna(), f'home_team_history_opponent_goal_{i}'] = median_leagues_id_goals.loc[df.loc[df[f'home_team_history_opponent_goal_{i}'].isna(), f'home_team_history_league_id_{i}']].values\n df.loc[df[f'away_team_history_goal_{i}'].isna(), f'away_team_history_goal_{i}'] = median_leagues_id_goals.loc[df.loc[df[f'away_team_history_goal_{i}'].isna(), f'away_team_history_league_id_{i}']].values\n df.loc[df[f'away_team_history_opponent_goal_{i}'].isna(), f'away_team_history_opponent_goal_{i}'] = median_leagues_id_goals.loc[df.loc[df[f'away_team_history_opponent_goal_{i}'].isna(), f'away_team_history_league_id_{i}']].values\n\n return df","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:21:26.850963Z","iopub.execute_input":"2022-12-04T10:21:26.851483Z","iopub.status.idle":"2022-12-04T10:21:26.859909Z","shell.execute_reply.started":"2022-12-04T10:21:26.851426Z","shell.execute_reply":"2022-12-04T10:21:26.859074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = fill_na_history_goals(X_train, median_leagues_id_goals, drop_last_games_above_i)\nX_test = fill_na_history_goals(X_test, median_leagues_id_goals, drop_last_games_above_i)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:21:28.140605Z","iopub.execute_input":"2022-12-04T10:21:28.141070Z","iopub.status.idle":"2022-12-04T10:21:28.362762Z","shell.execute_reply.started":"2022-12-04T10:21:28.141032Z","shell.execute_reply":"2022-12-04T10:21:28.360670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Historical ratings features\n\nFill NaNs with the corresponding league_id median ratings\n\nSame logic as historical goals above","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame()\nfor i in range(1, drop_last_games_above_i + 1):\n # Train set\n df = pd.concat([df,\n # Train set \n pd.DataFrame(X_train[[f'home_team_history_league_id_{i}', f'home_team_history_rating_{i}']].values),\n pd.DataFrame(X_train[[f'home_team_history_league_id_{i}', f'home_team_history_opponent_rating_{i}']].values),\n pd.DataFrame(X_train[[f'away_team_history_league_id_{i}', f'away_team_history_rating_{i}']].values),\n pd.DataFrame(X_train[[f'away_team_history_league_id_{i}', f'away_team_history_opponent_rating_{i}']].values),\n # Test set\n pd.DataFrame(X_test[[f'home_team_history_league_id_{i}', f'home_team_history_rating_{i}']].values),\n pd.DataFrame(X_test[[f'home_team_history_league_id_{i}', f'home_team_history_opponent_rating_{i}']].values),\n pd.DataFrame(X_test[[f'away_team_history_league_id_{i}', f'away_team_history_rating_{i}']].values),\n pd.DataFrame(X_test[[f'away_team_history_league_id_{i}', f'away_team_history_opponent_rating_{i}']].values)\n ], \n axis=0)\n \ndf.columns = ['league_id', 'ratings']\nmedian_leagues_id_ratings = df.groupby('league_id').median()\nmedian_leagues_id_ratings.fillna(value=median_leagues_id_ratings.median(), inplace=True)\nmedian_leagues_id_ratings","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:21:48.089622Z","iopub.execute_input":"2022-12-04T10:21:48.090084Z","iopub.status.idle":"2022-12-04T10:21:48.687991Z","shell.execute_reply.started":"2022-12-04T10:21:48.090046Z","shell.execute_reply":"2022-12-04T10:21:48.686507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_na_history_ratings(df, median_leagues_id_ratings, drop_last_games_above_i):\n\n # Fill NaNs for past games (1-10)\n for i in range(1, drop_last_games_above_i + 1):\n df.loc[df[f'home_team_history_rating_{i}'].isna(), f'home_team_history_rating_{i}'] = median_leagues_id_ratings.loc[df.loc[df[f'home_team_history_rating_{i}'].isna(), f'home_team_history_league_id_{i}']].values\n df.loc[df[f'home_team_history_opponent_rating_{i}'].isna(), f'home_team_history_opponent_rating_{i}'] = median_leagues_id_ratings.loc[df.loc[df[f'home_team_history_opponent_rating_{i}'].isna(), f'home_team_history_league_id_{i}']].values\n df.loc[df[f'away_team_history_rating_{i}'].isna(), f'away_team_history_rating_{i}'] = median_leagues_id_ratings.loc[df.loc[df[f'away_team_history_rating_{i}'].isna(), f'away_team_history_league_id_{i}']].values\n df.loc[df[f'away_team_history_opponent_rating_{i}'].isna(), f'away_team_history_opponent_rating_{i}'] = median_leagues_id_ratings.loc[df.loc[df[f'away_team_history_opponent_rating_{i}'].isna(), f'away_team_history_league_id_{i}']].values\n\n return df","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:21:56.721728Z","iopub.execute_input":"2022-12-04T10:21:56.722138Z","iopub.status.idle":"2022-12-04T10:21:56.730272Z","shell.execute_reply.started":"2022-12-04T10:21:56.722106Z","shell.execute_reply":"2022-12-04T10:21:56.729420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = fill_na_history_ratings(X_train, median_leagues_id_ratings, drop_last_games_above_i)\nX_test = fill_na_history_ratings(X_test, median_leagues_id_ratings, drop_last_games_above_i)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:22:04.275374Z","iopub.execute_input":"2022-12-04T10:22:04.275925Z","iopub.status.idle":"2022-12-04T10:22:04.518199Z","shell.execute_reply.started":"2022-12-04T10:22:04.275884Z","shell.execute_reply":"2022-12-04T10:22:04.516783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Class weighting ?\n","metadata":{}},{"cell_type":"code","source":"from sklearn.utils.class_weight import compute_class_weight\n# Compute the class weights, we may use it in the loss function later, or not\nclass_weights = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train.values)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"code","source":"X_train[X_train['home_team_name'] == X_train['away_team_name']]","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:23:42.033066Z","iopub.execute_input":"2022-12-04T10:23:42.033506Z","iopub.status.idle":"2022-12-04T10:23:42.067349Z","shell.execute_reply.started":"2022-12-04T10:23:42.033472Z","shell.execute_reply":"2022-12-04T10:23:42.065388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There is two games where the home and away teams are the same -> We delete them\nto_remove = X_train[X_train['home_team_name'] == X_train['away_team_name']].index\nX_train.drop(to_remove, axis=0, inplace=True)\ny_train.drop(to_remove, axis=0, inplace=True)\n\n# Useless features to remove\nuseless_features = [\n 'home_team_name',\n 'away_team_name',\n 'league_name',\n]\n\n\ndef remove_useless_features(df, useless_features, drop_last_games_above_i):\n # Drop the useless features\n df.drop(useless_features, axis=1, inplace=True)\n # Drop the last games features\n lis = [f'_{i}' for i in range(drop_last_games_above_i+1,11)]\n df = df[[col for col in df.columns if all([l not in col for l in lis])]]\n return df\n\n\ndef nb_days_diff_goals_diff_ratings(df, drop_last_games_above_i):\n for i in range(1, drop_last_games_above_i + 1):\n\n # Goals difference between team and opponent team for last matches\n df[f'home_team_history_diff_goal_{i}'] = df[f'home_team_history_goal_{i}'] - df[f'home_team_history_opponent_goal_{i}']\n df[f'away_team_history_diff_goal_{i}'] = df[f'away_team_history_goal_{i}'] - df[f'away_team_history_opponent_goal_{i}']\n df.drop([f'home_team_history_goal_{i}', f'home_team_history_opponent_goal_{i}', f'away_team_history_goal_{i}', f'away_team_history_opponent_goal_{i}'], axis=1, inplace=True)\n\n # Ratings difference between team and opponent team for last matches\n df[f'home_team_history_diff_rating_{i}'] = df[f'home_team_history_rating_{i}'] - df[f'home_team_history_opponent_rating_{i}']\n df[f'away_team_history_diff_rating_{i}'] = df[f'away_team_history_rating_{i}'] - df[f'away_team_history_opponent_rating_{i}']\n df.drop([f'home_team_history_rating_{i}', f'home_team_history_opponent_rating_{i}', f'away_team_history_rating_{i}', f'away_team_history_opponent_rating_{i}'], axis=1, inplace=True)\n\n return df\n\ndef dtypes_to_float(df):\n # We assign each features to float type instead of match_date feature. \n non_float_features = df.select_dtypes(exclude=['float64'])\n for feature in non_float_features:\n if feature != 'match_date':\n df[feature] = df[feature].astype(np.float64)\n return df \n\ndef feature_engineering(df, useless_features, drop_last_games_above_i):\n df = remove_useless_features(df, useless_features, drop_last_games_above_i)\n df = nb_days_diff_goals_diff_ratings(df, drop_last_games_above_i)\n df = dtypes_to_float(df)\n return df","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:23:56.832629Z","iopub.execute_input":"2022-12-04T10:23:56.833091Z","iopub.status.idle":"2022-12-04T10:23:56.958979Z","shell.execute_reply.started":"2022-12-04T10:23:56.833057Z","shell.execute_reply":"2022-12-04T10:23:56.958276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = feature_engineering(X_train, useless_features, drop_last_games_above_i)\nX_test = feature_engineering(X_test, useless_features, drop_last_games_above_i)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:24:04.087707Z","iopub.execute_input":"2022-12-04T10:24:04.088136Z","iopub.status.idle":"2022-12-04T10:24:06.684058Z","shell.execute_reply.started":"2022-12-04T10:24:04.088109Z","shell.execute_reply":"2022-12-04T10:24:06.683149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check $X_{train}$","metadata":{}},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:24:22.307150Z","iopub.execute_input":"2022-12-04T10:24:22.307584Z","iopub.status.idle":"2022-12-04T10:24:22.367257Z","shell.execute_reply.started":"2022-12-04T10:24:22.307550Z","shell.execute_reply":"2022-12-04T10:24:22.364964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# LSTM (with PyTorch)\n\nTODO: Hyperparameter optimization","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, TensorDataset, ConcatDataset, DataLoader\nfrom torch import nn\nfrom torch.autograd import Variable\nfrom torch import optim\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\n\n# Device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint('Device: ', device)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:25:29.947573Z","iopub.execute_input":"2022-12-04T10:25:29.948575Z","iopub.status.idle":"2022-12-04T10:25:32.128427Z","shell.execute_reply.started":"2022-12-04T10:25:29.948542Z","shell.execute_reply":"2022-12-04T10:25:32.126562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # For the boolean or id features, we fill NaN with most frequent value\nbool_id_features = [feat for feat in X_train.columns if len(X_train[feat].dropna().unique()) == 2] + [feat for feat in X_train.columns if ('league' in feat or 'coach' in feat)]\nfor feat in bool_id_features:\n if feat != 'match_date':\n X_train[feat] = X_train[feat].fillna(value=X_train[feat].value_counts().index[0])\n X_test[feat] = X_test[feat].fillna(value=X_train[feat].value_counts().index[0])","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:25:45.866619Z","iopub.execute_input":"2022-12-04T10:25:45.867129Z","iopub.status.idle":"2022-12-04T10:25:46.550873Z","shell.execute_reply.started":"2022-12-04T10:25:45.867100Z","shell.execute_reply":"2022-12-04T10:25:46.549251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.isna().any().any()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:25:51.183118Z","iopub.execute_input":"2022-12-04T10:25:51.183636Z","iopub.status.idle":"2022-12-04T10:25:51.206265Z","shell.execute_reply.started":"2022-12-04T10:25:51.183598Z","shell.execute_reply":"2022-12-04T10:25:51.204748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.isna().any().any()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:25:58.314712Z","iopub.execute_input":"2022-12-04T10:25:58.315129Z","iopub.status.idle":"2022-12-04T10:25:58.331921Z","shell.execute_reply.started":"2022-12-04T10:25:58.315095Z","shell.execute_reply":"2022-12-04T10:25:58.330618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train/Validation split\n\nLet's split ou train in train and validation datasets with respect to the time series property.\n\nWe assume that 5000 games for validation (5% of the dataset) is enough, let's split:\n- train set: 95%\n- validation set: 5%","metadata":{}},{"cell_type":"code","source":"val_split = int(X_train.shape[0]*0.80)\nfig = go.Figure()\na = pd.concat([X_train['match_date'], pd.DataFrame(np.arange(X_train.shape[0]), index=X_train.index)], axis=1)\nfig.add_trace(go.Scatter(x=a['match_date'], y=a.iloc[:,1]))\nfig.add_vline(X_train['match_date'].iloc[val_split], name='validation_split')\nfig.update_layout(title='Nb of matches vs match date. Train until: {}'.format(X_train['match_date'].iloc[val_split]))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:26:20.271885Z","iopub.execute_input":"2022-12-04T10:26:20.272345Z","iopub.status.idle":"2022-12-04T10:26:20.811342Z","shell.execute_reply.started":"2022-12-04T10:26:20.272309Z","shell.execute_reply":"2022-12-04T10:26:20.809433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tr, y_tr = X_train.iloc[:val_split], y_train.iloc[:val_split]\nX_val, y_val = X_train.iloc[val_split:], y_train.iloc[val_split:]","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:26:34.747828Z","iopub.execute_input":"2022-12-04T10:26:34.749642Z","iopub.status.idle":"2022-12-04T10:26:34.756794Z","shell.execute_reply.started":"2022-12-04T10:26:34.749589Z","shell.execute_reply":"2022-12-04T10:26:34.755541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Processing input data\n\nThe input need to be in 3D: ($N$, $L$, $H_{in}$) with:\n- $N$: batch size (in the first place with the argument `batch_first=True`)\n- $L$: timesteps (the last games$_i$, $i \\in [10,1]$)\n- $H_{in}$: features dimension (ratings, nb_goals,...)\n\nHere is an example of the 3D input data to feed to the LSTM:\n
\n
\n
Source: This blog
\n
","metadata":{}},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"# The historical features, will be transformed into a 3D matrix\nhistorical_features = [\n # Home\n 'home_team_history_is_play_home',\n 'home_team_history_is_cup',\n 'home_team_history_diff_goal',\n 'home_team_history_diff_rating',\n # 'home_team_history_goal',\n # 'home_team_history_rating',\n 'home_team_history_coach',\n 'home_team_history_league_id',\n 'home_team_history_nb_days_match',\n\n # Away\n 'away_team_history_is_play_home',\n 'away_team_history_is_cup',\n 'away_team_history_diff_goal',\n 'away_team_history_diff_rating',\n # 'away_team_history_goal',\n # 'away_team_history_rating',\n 'away_team_history_coach',\n 'away_team_history_league_id',\n 'away_team_history_nb_days_match'\n]\n\n# The metadata_features, will be concatenated to the LSTM's output\nmetadata_features = ['league_id', 'is_cup', 'home_team_coach_id', 'away_team_coach_id']\n# metadata_features = ['league_id', 'is_cup']","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:32:34.987215Z","iopub.execute_input":"2022-12-04T10:32:34.987730Z","iopub.status.idle":"2022-12-04T10:32:34.995339Z","shell.execute_reply.started":"2022-12-04T10:32:34.987691Z","shell.execute_reply":"2022-12-04T10:32:34.993408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FootballPredDataset(Dataset):\n '''\n Custom dataset for the Football Match prediction. \n Inputs:\n - X: the features matrix to process\n - y: the labels correponding if train or validation dataset\n - timesteps: the number of last games taken into account\n - historical_features: X's features, will be transformed into a 3D matrix to be put into the LSTM\n - metadata_features: X's features, will be concatenated to the hidden states LSTM's output\n - transform: data transformation\n Outputs:\n - X_hist_3D: Tensor (N, L, H_hist), will be feed to the LSTM\n - X_metadata: Tensor (N, H_meta), will be concatenated to the LSTM's output hidden states\n '''\n\n def __init__(self, X, historical_features, metadata_features, y=None, timesteps=drop_last_games_above_i, transform='StandardScaler'):\n self.X = X\n self.timesteps = timesteps\n self.transform = transform\n\n self.X_hist_3D = torch.stack([\n torch.from_numpy(self.X[[f'{feat}_{i}' for i in range(1,self.timesteps+1)[::-1]]].values) for feat in historical_features\n ], \n dim=-1)\n \n self.X_metadata = torch.from_numpy(self.X[metadata_features].values)\n\n if transform == 'StandardScaler':\n self.X_hist_3D = (self.X_hist_3D - self.X_hist_3D.mean(axis=[0,1], keepdim=True)) / self.X_hist_3D.std(axis=[0,1], keepdim=True)\n self.X_metadata = (self.X_metadata - self.X_metadata.mean(axis=0, keepdim=True)) / self.X_metadata.std(axis=0, keepdim=True)\n \n if y is not None:\n self.y = torch.from_numpy(np.array(y))\n else:\n self.y = None\n\n def __len__(self):\n return self.X.shape[0]\n\n def __getitem__(self, idx):\n if self.y is not None:\n return self.X_hist_3D[idx, :, :], self.X_metadata[idx, :], self.y[idx]\n else:\n return self.X_hist_3D[idx, :, :], self.X_metadata[idx, :]","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:32:37.287692Z","iopub.execute_input":"2022-12-04T10:32:37.288135Z","iopub.status.idle":"2022-12-04T10:32:37.301765Z","shell.execute_reply.started":"2022-12-04T10:32:37.288098Z","shell.execute_reply":"2022-12-04T10:32:37.300755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Datasets\n\ntransform = 'StandardScaler'\n# transform = None\n\ntrain_dataset = FootballPredDataset(X=X_tr, y=y_tr, historical_features=historical_features, metadata_features=metadata_features, transform='StandardScaler')\nval_dataset = FootballPredDataset(X=X_val, y=y_val, historical_features=historical_features, metadata_features=metadata_features, transform='StandardScaler')\n\n# Dataloaders\nbatch_size = 96\nshuffle = False\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=shuffle)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=shuffle)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:32:45.913492Z","iopub.execute_input":"2022-12-04T10:32:45.914014Z","iopub.status.idle":"2022-12-04T10:32:46.354374Z","shell.execute_reply.started":"2022-12-04T10:32:45.913974Z","shell.execute_reply":"2022-12-04T10:32:46.352760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model\n\nHere is an example w.r.t pytorch variables names for our LSTM:\n\n
\n
\n
Source: StackOverflow
\n
\n\nWith:\n- depth: the number of layers (stacked LSTMs)\n- t: the timesteps dimension\n- input: our 3D matrix of historical features\n\nNote:\n- In PyTorch, there is no need to add a softmax activation for the output layer like in Keras.\n - Keras: \n `softmax(output)` + `loss='sparce_categorical_cross_entropy'`\n - Pytorch: `log_softmax(output)` + `NLLLoss()` = `output` + `CrossEntropyLoss()`\n\n- Here we use the `CrossEntropyLoss()`.\n ","metadata":{}},{"cell_type":"code","source":"class FootballPredNet(nn.Module):\n '''\n Network composed of an LSTM and potentially a concatenation of LSTM's hidden states output and our metadata. \n '''\n\n def __init__(self, input_dim, hidden_dim, n_layers=2, output_dim=3, len_metadata=0, bidirectional=False):\n # Parent class initialization\n super(FootballPredNet, self).__init__()\n\n # Number of historical features\n self.input_dim = input_dim\n # Number of hidden states dimension\n self.hidden_dim = hidden_dim\n # Boolean for metadata inclusion (concatenation with the LSTM outputs) -> 0 means no metadata\n self.len_metadata = len_metadata\n # Number of stacked LSTM\n self.n_layers = n_layers\n # Number of classes (always 3)\n self.output_dim = output_dim\n # Bi-LSTM or not\n self.bidirectional = bidirectional\n\n # LSTM layer\n self.lstm = nn.LSTM(input_dim, hidden_dim, n_layers, batch_first=True, bidirectional=bidirectional)\n # Batch Normalization Layer\n self.batch_norm = nn.BatchNorm1d((int(self.bidirectional)+1)*hidden_dim + len_metadata)\n # Tanh ? \n self.tanh = nn.Tanh()\n # # ReLU ?\n self.relu = nn.ReLU()\n # # Dropout ?\n self.dropout = nn.Dropout(p=0.2)\n # Dense layer\n self.dense = nn.Linear((int(self.bidirectional)+1)*hidden_dim + len_metadata, output_dim)\n\n def forward(self, x_lstm, x_meta):\n # Initialize hidden state\n h0 = torch.zeros((int(self.bidirectional)+1)*self.n_layers, x_lstm.size(0), self.hidden_dim).requires_grad_()\n h0 = h0.to(device)\n\n # Initialize cell state\n c0 = torch.zeros((int(self.bidirectional)+1)*self.n_layers, x_lstm.size(0), self.hidden_dim).requires_grad_()\n c0 = c0.to(device)\n\n # We need to detach as we are doing truncated backpropagation through time (BPTT)\n # If we don't, we'll backprop all the way to the start even after going through another batch\n out, (hn, cn) = self.lstm(x_lstm, (h0.detach(), c0.detach())) # out[id_batch,-1,:] == hn[-1,id_batch,:] !\n\n if self.len_metadata > 0:\n output = self.dense(self.tanh(self.batch_norm(self.dropout(torch.cat([out[:, -1, :], x_meta], dim=-1)))))\n else:\n output = self.dense(self.batch_norm(out[:, -1, :])) # out[:, -1, :] -> just want last time step hidden states!\n # output = self.dense(out[:, -1, :]) # out[:, -1, :] -> just want last time step hidden states!\n\n # output = torch.log_softmax(output, dim=-1) # To remove if we use CrossEntropyLoss()\n return output","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:33:34.666556Z","iopub.execute_input":"2022-12-04T10:33:34.667022Z","iopub.status.idle":"2022-12-04T10:33:34.685411Z","shell.execute_reply.started":"2022-12-04T10:33:34.666970Z","shell.execute_reply":"2022-12-04T10:33:34.683929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LSTM parameters\n# Input dimension (nb of historical features)\ninput_dim = train_dataset[0][0].shape[-1]\n# Hidden states dimension\nhidden_dim = 12\n# Numbers of layers (stacked LSTMs)\nn_layers = 1\n\n# Metadata to concatenate, or not\ninclude_metadata = True\nif include_metadata:\n len_metadata = len(metadata_features)\nelse:\n len_metadata = 0\n\n# Bi-LSTM, or not\nbidirectional = True\n\n# Network instantiation\nmodel = FootballPredNet(input_dim, hidden_dim, n_layers=n_layers, len_metadata=len_metadata, bidirectional=bidirectional)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:33:46.435816Z","iopub.execute_input":"2022-12-04T10:33:46.436153Z","iopub.status.idle":"2022-12-04T10:33:46.451558Z","shell.execute_reply.started":"2022-12-04T10:33:46.436123Z","shell.execute_reply":"2022-12-04T10:33:46.450530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training loop","metadata":{}},{"cell_type":"code","source":"# Nb of epochs\nepochs = 100\n\n# Loss function, with class_weights, or not\nuse_class_weights = False\nif use_class_weights:\n class_weights_tensor = torch.from_numpy(class_weights)\n class_weights_tensor = class_weights_tensor.to(device)\n criterion = nn.CrossEntropyLoss(weight=class_weights_tensor.float())\nelse:\n criterion = nn.CrossEntropyLoss()\n# criterion = nn.NLLLoss()\n\n# Learning rate\nlr = 0.001\n\n# Optimizer\n# optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.8)\n# optimizer = optim.RMSprop(model.parameters(), lr=lr)\noptimizer = optim.Adam(model.parameters(), lr=lr)\n\n# Scheduler (learning rate)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:34:07.830159Z","iopub.execute_input":"2022-12-04T10:34:07.830547Z","iopub.status.idle":"2022-12-04T10:34:07.839072Z","shell.execute_reply.started":"2022-12-04T10:34:07.830515Z","shell.execute_reply":"2022-12-04T10:34:07.837554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_losses = []\nvalid_losses = []\n\nfor i in tqdm(range(epochs), 'Epochs'):\n print('Epoch {}:'.format(i+1))\n\n # Initialize the current epoch's losses\n train_loss = 0.\n valid_loss = 0.\n\n # Training\n model.train()\n for X_lstm, X_meta, y in tqdm(train_loader, 'Train Batches'):\n # for X_lstm, X_meta, y in train_loader:\n # print(X_lstm.size())\n X_lstm, X_meta, y = X_lstm.float().to(device), X_meta.float().to(device), y.to(device)\n logpreds = model(X_lstm, X_meta)\n loss = criterion(logpreds, y)\n optimizer.zero_grad()\n loss.backward() # dw, db\n optimizer.step() # w = w - lr*dw\n train_loss += loss.item()\n\n # Validation\n model.eval() # Dropout, Batch Norm\n for X_lstm, X_meta, y in tqdm(val_loader, 'Valid Batches'):\n X_lstm, X_meta, y = X_lstm.float().to(device), X_meta.float().to(device), y.to(device)\n logpreds = model(X_lstm, X_meta)\n loss = criterion(logpreds, y)\n valid_loss += loss.item()\n\n # Get the average epoch's losses\n avg_train_loss = train_loss/len(train_loader)\n avg_valid_loss = valid_loss/len(val_loader)\n\n # Append to the losses list\n train_losses.append(avg_train_loss)\n valid_losses.append(avg_valid_loss)\n\n # Step for the learning rate scheduler\n scheduler.step(avg_train_loss)\n\n print('Train Loss: {:.5f}'.format(avg_train_loss))\n print('Valid Loss: {:.5f}'.format(avg_valid_loss))\n print('------------------')","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:34:18.898165Z","iopub.execute_input":"2022-12-04T10:34:18.898575Z","iopub.status.idle":"2022-12-04T10:47:59.413352Z","shell.execute_reply.started":"2022-12-04T10:34:18.898546Z","shell.execute_reply":"2022-12-04T10:47:59.412196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's print the train/Valid losses vs epochs\nfig = go.Figure()\nfig.add_trace(go.Scatter(x=np.arange(epochs), y=train_losses, name='Train loss'))\nfig.add_trace(go.Scatter(x=np.arange(epochs), y=valid_losses, name='Val loss'))\nfig.update_layout(title='Train & Valid losses vs Epochs', xaxis_title='Epochs')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:47:59.415631Z","iopub.execute_input":"2022-12-04T10:47:59.416517Z","iopub.status.idle":"2022-12-04T10:47:59.453421Z","shell.execute_reply.started":"2022-12-04T10:47:59.416478Z","shell.execute_reply":"2022-12-04T10:47:59.452219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Eval on test","metadata":{}},{"cell_type":"code","source":"# Test Dataset\ntest_dataset = FootballPredDataset(X_test, historical_features=historical_features, metadata_features=metadata_features, transform='StandardScaler')\n\n# Test Dataloader\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:47:59.454824Z","iopub.execute_input":"2022-12-04T10:47:59.455243Z","iopub.status.idle":"2022-12-04T10:47:59.684311Z","shell.execute_reply.started":"2022-12-04T10:47:59.455207Z","shell.execute_reply":"2022-12-04T10:47:59.683024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval() # Dropout, Batch Norm\n\n# Initialize predictions\nalllogpreds = torch.Tensor().cuda()\n\nwith torch.no_grad():\n for X_lstm, X_meta in tqdm(test_loader, 'Test Batches'):\n X_lstm, X_meta = X_lstm.to(device), X_meta.to(device)\n logpreds = model(X_lstm.float(), X_meta.float())\n alllogpreds = torch.cat((alllogpreds, logpreds))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:47:59.686830Z","iopub.execute_input":"2022-12-04T10:47:59.687985Z","iopub.status.idle":"2022-12-04T10:47:59.771244Z","shell.execute_reply.started":"2022-12-04T10:47:59.687908Z","shell.execute_reply":"2022-12-04T10:47:59.769424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the probabilities\nprobs = torch.softmax(alllogpreds, dim=-1)\n\n# Create dataframe for submission\n# submission = pd.DataFrame(probs.detach().cpu().numpy(), index=X_test.index, columns=['home', 'draw', 'away']).reset_index()\n\n# Write into a csv for Kaggle submission\n# submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T10:47:59.772908Z","iopub.status.idle":"2022-12-04T10:47:59.773388Z","shell.execute_reply.started":"2022-12-04T10:47:59.773138Z","shell.execute_reply":"2022-12-04T10:47:59.773170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}