{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Project description\n\nMy initial goal was to create logistic regression model with somewhat similiar score to public LSTM's, i.e. <= 1. While I didn't successed in breaking 1 `log-loss` public LB value, I managed to drastically improve log. regssion model's score 1.02505 **[1]** from to 1.01058. My main goal of this notebook was to train model without imputing missing values. In depth comments how I dealt with missing values can be found in this comment .\n\nMain observations:\n\n* I trained models without imputing missing values,\n* Used cross-validation to identify best features to use for training log. regression model.\n* **baseline** `goals` + `days since last match` -> 1.02340,\n* **attempt #1** `goals` + `days since last match + ratings` -> 1.01145,\n* **attempt #2** `goals` + `days since last match + ratings + is cup` -> 1.01058,\n* **attempt #3** `goals` + `days since last match + ratings + is same league` -> 1.01089,\n* **final** `goals` + `days since last match + ratings + is cup + is same league` -> 1.01059.\n\nFeatures `is play at home` and `same coach` didn't provide meaningful `log-loss` score improvements.\n\nThis notebook has a really long- run time because it does both cross-validation and trains the best models. Without EDA and CV (just training best model) this notebook is rather fast.\n\n**[1]** https://www.kaggle.com/code/uzdavinys/xg-expected-goals-with-simple-sklearn-models first attempt using log. regression.","metadata":{}},{"cell_type":"code","source":"# linear algebra\nimport numpy as np\n# data processing\nimport pandas as pd \n# static data visualization\nimport matplotlib.pyplot as plt\n# for statistical visualizations\nimport seaborn as sns\n# interactive visualization\nfrom plotly.subplots import make_subplots\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n# for generating combinations\nfrom itertools import combinations\n\n# set default plotly theme\nimport plotly.io as pio\npio.templates.default = \"simple_white\"\n\n# for scalling features (2 options)\nfrom sklearn.preprocessing import MinMaxScaler, RobustScaler\n# main metric for copetition\nfrom sklearn.metrics import log_loss\n\n# bayes models\nfrom sklearn.naive_bayes import BernoulliNB, GaussianNB\n# linear models\nfrom sklearn.linear_model import LogisticRegression\n\n# for multiclass clasification\nfrom sklearn.multiclass import OneVsRestClassifier\n\n# for K-Folds cross-validation\nfrom sklearn.model_selection import KFold\n\n# for monitorinc process speeeds\nimport time\n\n# contants\nRANDOM_STATE = 42\n\n# plot style\nplt.style.use('ggplot')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-04-29T14:55:25.019969Z","iopub.execute_input":"2022-04-29T14:55:25.020684Z","iopub.status.idle":"2022-04-29T14:55:29.053440Z","shell.execute_reply.started":"2022-04-29T14:55:25.020556Z","shell.execute_reply":"2022-04-29T14:55:29.052269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **1.** Load and format data","metadata":{}},{"cell_type":"code","source":"# load the data\ndf_train = pd.read_csv('/kaggle/input/football-match-probability-prediction/train.csv', low_memory = False)\ndf_test = pd.read_csv('/kaggle/input/football-match-probability-prediction/test.csv', low_memory = False)\ndf_scores = pd.read_csv('/kaggle/input/football-match-probability-prediction/train_target_and_scores.csv', low_memory = False)\nsubmission = pd.read_csv('/kaggle/input/football-match-probability-prediction/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:13.074063Z","iopub.execute_input":"2022-04-29T14:56:13.074362Z","iopub.status.idle":"2022-04-29T14:56:28.223287Z","shell.execute_reply.started":"2022-04-29T14:56:13.074329Z","shell.execute_reply":"2022-04-29T14:56:28.222324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, df_name, verbose=True):\n    \"\"\"\n    This function changes pandas numerical dtypes (reduces bit size if possible)\n    to reduce memory usage\n    \n    :param df: pandas DataFrame\n    :param df_name: str, name of DataFrame\n    :param verbose: bool, if True prints out message of how much memory usage was reduced\n        \n    :return:  pandas DataFrame       \n    \"\"\"\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int64)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float64)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    # calculate memory after reduction\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        # reduced memory usage in percent\n        diff_pst = 100 * (start_mem - end_mem) / start_mem\n        msg = f'{df_name} mem. usage decreased to {end_mem:5.2f} Mb ({diff_pst:.1f}% reduction)'\n        print(msg)\n    return df","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-04-29T14:56:28.225184Z","iopub.execute_input":"2022-04-29T14:56:28.225412Z","iopub.status.idle":"2022-04-29T14:56:28.242288Z","shell.execute_reply.started":"2022-04-29T14:56:28.225384Z","shell.execute_reply":"2022-04-29T14:56:28.241203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reduce memory usage if available\ndf_train = reduce_mem_usage(df_train, \"df_train\")\ndf_test = reduce_mem_usage(df_test, \"df_test\")","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:28.243598Z","iopub.execute_input":"2022-04-29T14:56:28.244386Z","iopub.status.idle":"2022-04-29T14:56:34.737793Z","shell.execute_reply.started":"2022-04-29T14:56:28.244348Z","shell.execute_reply":"2022-04-29T14:56:34.736884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add home and away scores\ndf_scores['home'] = df_scores.score.apply(lambda x: x.split(\"-\")[0]).astype(int)\ndf_scores['away'] = df_scores.score.apply(lambda x: x.split(\"-\")[1]).astype(int)\ndf_scores.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:34.739769Z","iopub.execute_input":"2022-04-29T14:56:34.740072Z","iopub.status.idle":"2022-04-29T14:56:34.875337Z","shell.execute_reply.started":"2022-04-29T14:56:34.740033Z","shell.execute_reply":"2022-04-29T14:56:34.874315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create DataFrames to store outcomes\ny_train = pd.DataFrame()\n\n# one-hot-encode target\nfor side in ['home', 'away', 'draw']:\n    y_train[side] = (df_train.target == side).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:34.878048Z","iopub.execute_input":"2022-04-29T14:56:34.878855Z","iopub.status.idle":"2022-04-29T14:56:34.964362Z","shell.execute_reply.started":"2022-04-29T14:56:34.878798Z","shell.execute_reply":"2022-04-29T14:56:34.963470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get column names with string \"date\"\ncols = [_ for _ in df_train.columns if 'date' in _]\n\n# convert to datetime\nfor c in cols:\n    df_train.loc[:, c] = pd.to_datetime(df_train.loc[:, c])\n    df_test.loc[:, c] = pd.to_datetime(df_test.loc[:, c])","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:34.965657Z","iopub.execute_input":"2022-04-29T14:56:34.965937Z","iopub.status.idle":"2022-04-29T14:56:36.710442Z","shell.execute_reply.started":"2022-04-29T14:56:34.965885Z","shell.execute_reply":"2022-04-29T14:56:36.709376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# only one value is missing regarding if match was cup of not\nprint(df_train.is_cup.isna().sum())\n# all values are not missing for test dataset\nprint(df_test.is_cup.isna().sum())\n\n# let's assume that if it is unknown whatever game was cup, make it 0 (it is more likely that it wasn't a cup game)\ndf_train.is_cup.fillna(False, inplace=True)\n# just in case due the same for test dataset\ndf_test.is_cup.fillna(False, inplace=True)\n\n# convert features to integeres\ndf_train.is_cup = df_train.is_cup.astype(int)\ndf_test.is_cup = df_test.is_cup.astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:36.711583Z","iopub.execute_input":"2022-04-29T14:56:36.711798Z","iopub.status.idle":"2022-04-29T14:56:36.758268Z","shell.execute_reply.started":"2022-04-29T14:56:36.711772Z","shell.execute_reply":"2022-04-29T14:56:36.757254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **2.** Missing data on `n` last historical games\n\nInformation about historcial hames can be split into 2 groups:\n\n* `home_team_history_`,\n* `away_team_history_`,\n\nand 9 sub-groups:\n\n* `match_date`,\n* `is_play_home`,\n* `is_cup`,\n* `goal`,\n* `opponent_goal`,\n* `rating`,\n* `opponent_rating`,\n* `coach`,\n* `league_id`.\n\nFor `n=10` games missing values should be analyzed for **180** features.","metadata":{}},{"cell_type":"code","source":"# create DataFrame with information whatever value is missing or not for those 180 features\ndf_train_nan_all = df_train[df_train.columns[10:]].isna()\ndf_test_nan_all = df_test[df_test.columns[9:]].isna()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:36.759545Z","iopub.execute_input":"2022-04-29T14:56:36.760241Z","iopub.status.idle":"2022-04-29T14:56:36.927025Z","shell.execute_reply.started":"2022-04-29T14:56:36.760205Z","shell.execute_reply":"2022-04-29T14:56:36.926311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_NaN_values_per_match(df, cols):\n    \"\"\"\n    This function count missing data entries per each historic game for each side.\n    \n    :param df: pandas DataFrame\n    :param cols: list, column names\n        \n    :return:  pandas DataFrame\n    \"\"\"\n    # create new empty DataFrame to store results\n    df_na = pd.DataFrame()\n    \n    # count missing values per n historical matches\n    for n in range(1, 11):\n        for side in ['home', 'away']:\n            # generate column names\n            _cols = [f\"{side}{_}{n}\" for _ in cols]\n            # count NaN values for home team historical n games and return 1 if no NaN value were found, else 0\n            df_na.loc[:, f'NaN_{side}_team_{n}'] = (df.loc[:, _cols].sum(axis=1) == 0).astype(int)\n            \n    # check whatever for game n information is known for both home and away teams\n    for n in range(1, 11):\n        df_na.loc[:, f'NaN_{n}'] = df_na.loc[:, f'NaN_home_team_{n}'] * df_na.loc[:, f'NaN_away_team_{n}']\n    # count number of know games for each side\n    df_na.loc[:, 'NaN_home_cnt'] = df_na.loc[:, [f'NaN_home_team_{n}' for n in range(1, 11)]].sum(axis=1)\n    df_na.loc[:, 'NaN_away_cnt'] = df_na.loc[:, [f'NaN_away_team_{n}' for n in range(1, 11)]].sum(axis=1)\n    # count min known games, e.g. if we have info about 3 last home games but only 2 for away, we have information\n    # only for 2 last games for both teams\n    df_na.loc[:, 'NaN_min_cnt'] = df_na.loc[:, ['NaN_home_cnt', 'NaN_away_cnt']].min(axis=1)\n    \n    # return DataFrame with NaN stats\n    return df_na","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-04-29T14:56:36.928486Z","iopub.execute_input":"2022-04-29T14:56:36.928960Z","iopub.status.idle":"2022-04-29T14:56:36.938957Z","shell.execute_reply.started":"2022-04-29T14:56:36.928911Z","shell.execute_reply":"2022-04-29T14:56:36.938295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_pie_char(df_train_nan, df_test_nan, cond=True):\n    \"\"\"\n    This function return plotly chart with missing data proportions.\n    \n    :param df_train_nan: pandas DataFrame\n    :param df_test_nan: pandas DataFrame\n    :param cond: boolean\n        \n    :return:  pandas DataFrame\n    \"\"\"\n    # there are 2 ways to estimate number of not missing games \n    if cond:\n        # figure title\n        t = 'Missing data analysis (same games)'\n        # number of games with no missing data\n        df_plot = df_train_nan.iloc[:, :10].sum(axis=1).value_counts()\n        # add info from test dataset\n        df_plot = pd.concat([df_plot, df_test_nan.iloc[:, :10].sum(axis=1).value_counts()], axis=1)\n    else:\n        # figure title\n        t = 'Missing data analysis (shifted games)'\n        # count number of games with no missing data\n        df_plot = df_train_nan['NaN_min_cnt'].value_counts()\n        # add info from test dataset\n        df_plot = pd.concat([df_plot, df_test_nan['NaN_min_cnt'].value_counts()], axis=1)\n        \n    # add column names\n    df_plot.columns = ['train', 'test']\n\n    # create figure object\n    fig = make_subplots(rows=1, cols=2,\n                        specs=[[{\"type\": \"pie\"}, {\"type\": \"pie\"}]],\n                        subplot_titles=(\"Train dataset\", \"Test dataset\"))\n\n    # add train dataset stats\n    fig.add_trace(go.Pie(\n        values=df_plot['train'],\n        labels=10-df_plot.index,\n        hovertemplate=\"%{label} missing games<br>%{value:.0f} matches<br>%{percent}<extra></extra>\",\n        rotation=-45,\n        domain=dict(x=[0, 0.5])), \n        row=1, col=1)\n\n    # add train dataset stats\n    fig.add_trace(go.Pie(\n        values=df_plot['test'],\n        labels=10-df_plot.index,\n        hovertemplate=\"%{label} missing games<br>%{value:.0f} matches<br>%{percent}<extra></extra>\",\n        rotation=-45,\n        domain=dict(x=[0, 0.5])), \n        row=1, col=2)\n\n    # add title to the figure\n    fig.update_layout(title=t,\n                      title_x=0.5,\n                      legend_title=\"Missing games count\",)\n    \n    # return figure object\n    return fig","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-04-29T14:56:36.942153Z","iopub.execute_input":"2022-04-29T14:56:36.942635Z","iopub.status.idle":"2022-04-29T14:56:36.963351Z","shell.execute_reply.started":"2022-04-29T14:56:36.942585Z","shell.execute_reply":"2022-04-29T14:56:36.962214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_nan_cnt_matrix(df, _fig_title='Placeholder'):\n    \"\"\"\n    This function return plotly chart which compared counts with 2 conditions:\n        - left chart: matches which have at least equal n games with no missing data,\n        - right chart: counts matches with n and m missing data.\n    \n    :param df: pandas DataFrame\n    :param _fig_title: string\n        \n    :return:  pandas DataFrame\n    \"\"\"\n    # generate identity matrix\n    _mat_left = pd.DataFrame()\n    # count pairs\n    _mat_left['pair_cnt'] = df.iloc[:, :10].sum(axis=1).value_counts()\n    _mat_left['NaN_home_cnt'] = _mat_left.index\n    _mat_left['NaN_away_cnt'] = _mat_left.index\n    _mat_left = _mat_left.sort_values(by=\"NaN_home_cnt\")\n\n    # pivot table for heatmap\n    _mat_left = _mat_left.pivot(index='NaN_home_cnt', columns='NaN_away_cnt', values='pair_cnt').fillna(0).astype(int)\n\n    # generate left figure\n    fig_left = px.imshow(_mat_left.values,\n                          text_auto=True,\n                          aspect=\"auto\",\n                          labels=dict(x=\"data available for n matches at home\",\n                                      y=\"data available for n matches at home\",\n                                      color=\"Matches count\"))\n\n    # generate pairs for right chart\n    _mat_right = df.groupby(['NaN_home_cnt', 'NaN_away_cnt'])['NaN_min_cnt'].count().reset_index()\n    _mat_right.columns = ['NaN_home_cnt', 'NaN_away_cnt', 'pair_cnt']\n\n    # pivot table for heatmap\n    _mat_right = _mat_right.pivot(index='NaN_home_cnt', columns='NaN_away_cnt', values='pair_cnt') \n\n\n\n    # generate right figure\n    fig_right = px.imshow(_mat_right.values,\n                          text_auto=True,\n                          aspect=\"auto\",\n                          labels=dict(x=\"data available for n matches at home\",\n                                      y=\"data available for n matches at home\",\n                                      color=\"Matches count\"))\n\n    #Create a 1x2 subplot\n    fig = make_subplots(rows=1, cols=2) \n\n    # Get the Express fig broken down as traces and add the traces to the proper plot within in the subplot\n    fig.append_trace(fig_left[\"data\"][0], row=1, col=1)\n    fig.append_trace(fig_right[\"data\"][0], row=1, col=2)\n\n    # add figure title\n    fig.update_layout(title=_fig_title)\n\n    # update axis labels\n    for c in range(1, 3):\n        fig.update_xaxes(title_text=\"data available for n matches at home\", row=1, col=c)\n        fig.update_yaxes(title_text=\"data available for n matches at away\", row=1, col=c)\n        \n    # return plotly object\n    return fig","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-04-29T14:56:36.964828Z","iopub.execute_input":"2022-04-29T14:56:36.965168Z","iopub.status.idle":"2022-04-29T14:56:36.987391Z","shell.execute_reply.started":"2022-04-29T14:56:36.965127Z","shell.execute_reply":"2022-04-29T14:56:36.986162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **2.1** Best case senario\n\nIn the very best case scenario we would like to know all 9 features about home's and away's last 10 matches.","metadata":{}},{"cell_type":"code","source":"# column names to count per each match\ncols = [\"_team_history_match_date_\", \"_team_history_is_play_home_\", \"_team_history_is_cup_\",\n        \"_team_history_goal_\", \"_team_history_opponent_goal_\",\n        \"_team_history_rating_\", \"_team_history_opponent_rating_\",\n        \"_team_history_coach_\", \"_team_history_league_id_\"]\n\n# summarize NaN values\ndf_train_nan = count_NaN_values_per_match(df_train_nan_all, cols)\ndf_test_nan = count_NaN_values_per_match(df_test_nan_all, cols)\n\n# visualize distribution\nplot_pie_char(df_train_nan.iloc[:, -13:], df_test_nan.iloc[:, -13:])","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:36.988542Z","iopub.execute_input":"2022-04-29T14:56:36.989658Z","iopub.status.idle":"2022-04-29T14:56:37.586127Z","shell.execute_reply.started":"2022-04-29T14:56:36.989591Z","shell.execute_reply":"2022-04-29T14:56:37.585109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_nan_cnt_matrix(df_train_nan.iloc[:, -13:], 'Train dataset, Best case scenario')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:37.587830Z","iopub.execute_input":"2022-04-29T14:56:37.588250Z","iopub.status.idle":"2022-04-29T14:56:37.798142Z","shell.execute_reply.started":"2022-04-29T14:56:37.588213Z","shell.execute_reply":"2022-04-29T14:56:37.797155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_nan_cnt_matrix(df_test_nan.iloc[:, -13:], 'Test dataset, Best case scenario')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:37.799436Z","iopub.execute_input":"2022-04-29T14:56:37.799656Z","iopub.status.idle":"2022-04-29T14:56:37.957990Z","shell.execute_reply.started":"2022-04-29T14:56:37.799629Z","shell.execute_reply":"2022-04-29T14:56:37.956970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **2.2** Worst case senario\n\nIn the worst case scenario, we would like to know number of goals scored (hence outcome) last n games.","metadata":{}},{"cell_type":"code","source":"# column names to count per each match\ncols = [\"_team_history_goal_\", \"_team_history_opponent_goal_\"]\n\n# summarize NaN values\ndf_train_nan = count_NaN_values_per_match(df_train_nan_all, cols)\ndf_test_nan = count_NaN_values_per_match(df_test_nan_all, cols)\n\n# visualize distribution\nplot_pie_char(df_train_nan.iloc[:, -13:], df_test_nan.iloc[:, -13:])","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:37.959276Z","iopub.execute_input":"2022-04-29T14:56:37.959526Z","iopub.status.idle":"2022-04-29T14:56:38.235688Z","shell.execute_reply.started":"2022-04-29T14:56:37.959497Z","shell.execute_reply":"2022-04-29T14:56:38.234886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_nan_cnt_matrix(df_train_nan.iloc[:, -13:], 'Train dataset, Worst case scenario')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T16:48:52.202569Z","iopub.execute_input":"2022-04-29T16:48:52.202818Z","iopub.status.idle":"2022-04-29T16:48:52.371192Z","shell.execute_reply.started":"2022-04-29T16:48:52.202787Z","shell.execute_reply":"2022-04-29T16:48:52.370361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_nan_cnt_matrix(df_test_nan.iloc[:, -13:], 'Test dataset, Worst case scenario')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:38.398442Z","iopub.execute_input":"2022-04-29T14:56:38.398671Z","iopub.status.idle":"2022-04-29T14:56:38.556763Z","shell.execute_reply.started":"2022-04-29T14:56:38.398641Z","shell.execute_reply":"2022-04-29T14:56:38.555794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compare 2 scenarios affects on proportion of missing data for train and test samples.","metadata":{}},{"cell_type":"code","source":"# column names for difference scenarios\nscenario_cols = {\"best\":  [\"_team_history_match_date_\", \"_team_history_is_play_home_\", \"_team_history_is_cup_\",\n                           \"_team_history_goal_\", \"_team_history_opponent_goal_\",\n                           \"_team_history_rating_\", \"_team_history_opponent_rating_\",\n                           \"_team_history_coach_\", \"_team_history_league_id_\"],\n                 \"worst\": [\"_team_history_goal_\", \"_team_history_opponent_goal_\"]}\n\n# empty DataFrame to comparare different\ndf_nan = pd.DataFrame()\n\n# iterate over scenarios\nfor scen in scenario_cols.keys():\n    # columns to select for scenario\n    cols = scenario_cols[scen]\n    \n    # summarize NaN values\n    df_train_nan = count_NaN_values_per_match(df_train_nan_all, cols).iloc[:, -13:]\n    df_test_nan = count_NaN_values_per_match(df_test_nan_all, cols).iloc[:, -13:]\n    \n    # number of games with no missing data\n    _df = df_train_nan.iloc[:, :10].sum(axis=1).value_counts()\n    # add info from test dataset\n    _df = pd.concat([_df, df_test_nan.iloc[:, :10].sum(axis=1).value_counts()], axis=1).reset_index()\n    \n    # add column names\n    _df.columns = ['na_cnt', 'train', 'test']\n    \n    # sort by nan count\n    _df.sort_values(by='na_cnt', inplace=True, ascending=False)\n    \n    # count cumsum\n    _df.loc[:, ['train', 'test']] = _df[['train', 'test']].cumsum()\n    \n    # add scenario name\n    _df['scenario'] = scen\n    \n    # update results\n    df_nan = pd.concat([df_nan, _df]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:38.558267Z","iopub.execute_input":"2022-04-29T14:56:38.558491Z","iopub.status.idle":"2022-04-29T14:56:39.132237Z","shell.execute_reply.started":"2022-04-29T14:56:38.558464Z","shell.execute_reply":"2022-04-29T14:56:39.131156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# colors\ncolors = ['royalblue', 'firebrick']\n\n# create figure object\nfig = go.Figure()\n\nfor i, s in enumerate(df_nan.scenario.unique()):\n    # select data\n    df_plot = df_nan.loc[df_nan.scenario == s]\n    # get train data\n    fig.add_trace(\n        go.Scatter(x=df_plot.na_cnt,\n                   y=round(df_plot.train/len(df_train_nan_all)*100, 2),\n                   line=dict(color=colors[i], width=4),\n                   name=f'{s} scen. train dataset')\n    )\n    # get test data\n    fig.add_trace(\n        go.Scatter(x=df_plot.na_cnt,\n                   y=round(df_plot.test/len(df_test_nan_all)*100, 2),\n                   line=dict(color=colors[i], width=4, dash='dash'),\n                   name=f'{s} scen. test dataset')\n    )\n    \n# Edit the layout\nfig.update_layout(title=\"Proportion analysis\",\n                  xaxis_title='Known matches count',\n                  yaxis_title='Proportion (%)',\n                  hovermode=\"x unified\")\nfig","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:39.133724Z","iopub.execute_input":"2022-04-29T14:56:39.133976Z","iopub.status.idle":"2022-04-29T14:56:39.178395Z","shell.execute_reply.started":"2022-04-29T14:56:39.133945Z","shell.execute_reply":"2022-04-29T14:56:39.177756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best columns to select\ncols = [\"_team_history_match_date_\", \"_team_history_goal_\", \"_team_history_opponent_goal_\",\n        \"_team_history_rating_\", \"_team_history_opponent_rating_\",]\n\n# get whatever n-th game has required colun values\ndf_train_nan = count_NaN_values_per_match(df_train_nan_all, cols)\ndf_test_nan = count_NaN_values_per_match(df_test_nan_all, cols)\n\n# recount known last n matches streak\nfor df in [df_train_nan, df_test_nan]:\n    # home games\n    df.iloc[:, 0:20:2] = df.iloc[:, 0:20:2].cumsum(axis=1)\n    # away games\n    df.iloc[:, 1:20:2] = df.iloc[:, 1:20:2].cumsum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:39.179440Z","iopub.execute_input":"2022-04-29T14:56:39.179679Z","iopub.status.idle":"2022-04-29T14:56:39.532151Z","shell.execute_reply.started":"2022-04-29T14:56:39.179647Z","shell.execute_reply":"2022-04-29T14:56:39.531241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **2.3** Different scenarios\n\nUsing combinatorics analyze all possible combiantions of 9 sub-group columns and their impact on missing data. (I used brute force approach)","metadata":{}},{"cell_type":"code","source":"# generate all column combinations\ncols = [\"_team_history_match_date_\", \"_team_history_is_play_home_\", \"_team_history_is_cup_\",\n        \"_team_history_goal_\", \"_team_history_opponent_goal_\",\n        \"_team_history_rating_\", \"_team_history_opponent_rating_\",\n        \"_team_history_coach_\", \"_team_history_league_id_\"]\n\n\n# get all combinations\ncol_combs = []\nfor _ in range(1, len(cols)+1):\n    col_combs += [list(__) for __ in combinations(cols, _)]","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:39.535143Z","iopub.execute_input":"2022-04-29T14:56:39.535651Z","iopub.status.idle":"2022-04-29T14:56:39.542217Z","shell.execute_reply.started":"2022-04-29T14:56:39.535605Z","shell.execute_reply":"2022-04-29T14:56:39.541461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Comment following code cell to save time.","metadata":{}},{"cell_type":"code","source":"# start recording time\ns = time.time()\n\n# temp. data to store analysis results\n_data = list()\n\n# iterate over scenarios\nfor c in col_combs:    \n    # summarize NaN values\n    df_train_nan = count_NaN_values_per_match(df_train_nan_all, c)\n    df_test_nan = count_NaN_values_per_match(df_test_nan_all, c)\n    \n    # number of games with no missing data\n    _df = df_train_nan.iloc[:, :10].sum(axis=1).value_counts()\n    \n    # add info from test dataset\n    _df = pd.concat([_df, df_test_nan.iloc[:, :10].sum(axis=1).value_counts()], axis=1).reset_index()\n    \n    # add column names\n    _df.columns = ['na_cnt', 'train', 'test']\n    \n    # sort by nan count\n    _df.sort_values(by='na_cnt', inplace=True, ascending=False)\n    \n    # count values no data available\n    _data.append([len(c), c] + _df.loc[_df.na_cnt == 0, ['train', 'test']].values.tolist()[0] + _df.loc[_df.na_cnt == 10, ['train', 'test']].values.tolist()[0])\n    \nprint(f\"Feature exploration completed in {(time.time() - s):.1f} sec.\")","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:56:39.543311Z","iopub.execute_input":"2022-04-29T14:56:39.544001Z","iopub.status.idle":"2022-04-29T14:58:51.992083Z","shell.execute_reply.started":"2022-04-29T14:56:39.543859Z","shell.execute_reply":"2022-04-29T14:58:51.991134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some manual exploration was done\ndf_nan_meta = pd.DataFrame(_data, columns=['col_cnt', 'col_names', 'train_0_cnt', 'test_0_cnt', 'train_10_cnt', 'test_10_cnt'])\ndf_nan_meta.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:51.993251Z","iopub.execute_input":"2022-04-29T14:58:51.993463Z","iopub.status.idle":"2022-04-29T14:58:52.009718Z","shell.execute_reply.started":"2022-04-29T14:58:51.993437Z","shell.execute_reply":"2022-04-29T14:58:52.008738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count missing values for train dataset\ndf_nan_meta.train_10_cnt.value_counts().plot.bar()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:52.010962Z","iopub.execute_input":"2022-04-29T14:58:52.011217Z","iopub.status.idle":"2022-04-29T14:58:52.267773Z","shell.execute_reply.started":"2022-04-29T14:58:52.011190Z","shell.execute_reply":"2022-04-29T14:58:52.266806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(df_nan_meta.train_10_cnt.value_counts().index, df_nan_meta.train_10_cnt.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:52.269296Z","iopub.execute_input":"2022-04-29T14:58:52.269606Z","iopub.status.idle":"2022-04-29T14:58:52.499536Z","shell.execute_reply.started":"2022-04-29T14:58:52.269561Z","shell.execute_reply":"2022-04-29T14:58:52.498824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count missing values for train dataset\ndf_nan_meta.test_10_cnt.value_counts().plot.bar()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:52.501195Z","iopub.execute_input":"2022-04-29T14:58:52.501780Z","iopub.status.idle":"2022-04-29T14:58:52.713311Z","shell.execute_reply.started":"2022-04-29T14:58:52.501733Z","shell.execute_reply":"2022-04-29T14:58:52.712375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# quick plot to visualize results\nplt.scatter(df_nan_meta.test_10_cnt.value_counts().index, df_nan_meta.test_10_cnt.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:52.714736Z","iopub.execute_input":"2022-04-29T14:58:52.714995Z","iopub.status.idle":"2022-04-29T14:58:52.920564Z","shell.execute_reply.started":"2022-04-29T14:58:52.714965Z","shell.execute_reply":"2022-04-29T14:58:52.919689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most feature combinations produce 2-fold outcomes for both train and test datasets.","metadata":{}},{"cell_type":"markdown","source":"# **3.** Feature engineering\n\nDifferent features are created:\n\n* `goals` get number of goals","metadata":{}},{"cell_type":"code","source":"def select_features(df, cols):\n    \"\"\"\n    This function engineers features for training simple sklearn models.\n    \n    :param df: pandas DataFrame\n    :param cols: list, list of colums to transform\n        \n    :return:  pandas DataFrame\n    \"\"\"    \n    # get transformed features\n    X = df[['is_cup']].copy()\n    \n    # iterate through historic games\n    for n in range(1, 11):\n        if 'date' in cols:\n            # calculate days since last home and away games\n            X.loc[:, f'home_days_{n}'] = (df['match_date'] - df[f'home_team_history_match_date_{n}']).dt.days\n            X.loc[:, f'away_days_{n}'] = (df['match_date'] - df[f'away_team_history_match_date_{n}']).dt.days\n            \n        if 'goals' in cols:\n            for col_id in [f'home_team_history_goal_', f'home_team_history_opponent_goal_',\n                           f'away_team_history_goal_', f'away_team_history_opponent_goal_']:\n                X.loc[:, f\"{col_id}{n}\"] = df.loc[:, f\"{col_id}{n}\"]\n                \n        if 'goal diff' in cols:\n            # calculate goal difference\n            X.loc[:, f'home_goal_diff_{n}']  = df[f'home_team_history_goal_{n}'] - df[f'home_team_history_opponent_goal_{n}']\n            X.loc[:, f'away_goal_diff_{n}']  = df[f'away_team_history_goal_{n}'] - df[f'away_team_history_opponent_goal_{n}']\n            \n        if \"match outcome\" in cols:\n            # calculate goal difference\n            X.loc[:, f'home_goal_diff_{n}']  = df[f'home_team_history_goal_{n}'] - df[f'home_team_history_opponent_goal_{n}']\n            X.loc[:, f'away_goal_diff_{n}']  = df[f'away_team_history_goal_{n}'] - df[f'away_team_history_opponent_goal_{n}']\n            \n            # estimate game outcome from goal difference using manual one-hot-encoding\n            for side in ['home', 'away']:\n                # to same memory, draw outcome is not OHE, as 0 for home win and 0 for away win implicitly means that game's outcome was draw\n                X.loc[:, f'{side}_outcome_home_{n}'] = (X[f'home_goal_diff_{n}'] > 0).astype(int)\n                X.loc[:, f'{side}_outcome_away_{n}'] = (X[f'home_goal_diff_{n}'] < 0).astype(int)\n                \n            # delete column\n            del X[f'home_goal_diff_{n}'], X[f'away_goal_diff_{n}']\n            \n        if 'is cup' in cols:\n            # check if game was played in cup\n            X.loc[:, f\"home_team_history_is_cup_{n}\"] = df.loc[:, f\"home_team_history_is_cup_{n}\"]\n            X.loc[:, f\"away_team_history_is_cup_{n}\"] = df.loc[:, f\"away_team_history_is_cup_{n}\"]\n                \n        if 'ratings' in cols:\n            # add rakings column names\n            for col_id in ['home_team_history_rating_', 'home_team_history_opponent_rating_',\n                           'away_team_history_rating_', 'away_team_history_opponent_rating_']:\n                X.loc[:, f\"{col_id}{n}\"] = df.loc[:, f\"{col_id}{n}\"]\n                \n        if 'ratings v2' in cols:\n            # add modified rakings column names\n            # idea taken from https://www.kaggle.com/code/seraquevence/top-x-football-prob-prediction-lgbm-v01 \n            X[f'home_team_history_ELO_rating_{n}'] = 1/(1+10**((df.loc[:, f'home_team_history_opponent_rating_{n}'] - df.loc[:, f'home_team_history_rating_{n}'])/10))\n            X[f'away_team_history_ELO_rating_{n}'] = 1/(1+10**((df.loc[:, f'away_team_history_opponent_rating_{n}'] - df.loc[:, f'away_team_history_rating_{n}'])/10))\n            X[f'home_away_team_history_ELO_rating_{n}'] = 1/(1+10**((df.loc[:, f'away_team_history_rating_{n}'] - df.loc[:, f'home_team_history_rating_{n}'])/10))\n                \n        if 'play @home' in cols:\n            # add is play home column values\n            for col_id in [\"home_team_history_is_play_home_\", \"away_team_history_is_play_home_\"]:\n                X.loc[:, f\"{col_id}{n}\"] = df.loc[:, f\"{col_id}{n}\"]\n                \n        if 'same league' in cols:\n            # check if home and away team has historical games at that league\n            X.loc[:, f\"home_same_league_{n}\"] = (df.loc[:, 'league_id'] == df.loc[:, f\"home_team_history_league_id_{n}\"]).astype(int)\n            X.loc[:, f\"away_same_league_{n}\"] = (df.loc[:, 'league_id'] == df.loc[:, f\"away_team_history_league_id_{n}\"]).astype(int)\n            \n        if 'same coach' in cols:\n            # check if home and away team has their coach recently changed\n            X.loc[:, f\"home_same_coach_{n}\"] = (df.loc[:, 'home_team_coach_id'] == df.loc[:, f\"home_team_history_coach_{n}\"]).astype(int)\n            X.loc[:, f\"away_same_coach_{n}\"] = (df.loc[:, 'away_team_coach_id'] == df.loc[:, f\"away_team_history_coach_{n}\"]).astype(int)\n            \n        # to avoid defragmantation\n        X = X.copy()\n\n    # return selected features\n    return X","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:52.924491Z","iopub.execute_input":"2022-04-29T14:58:52.924748Z","iopub.status.idle":"2022-04-29T14:58:52.951534Z","shell.execute_reply.started":"2022-04-29T14:58:52.924717Z","shell.execute_reply":"2022-04-29T14:58:52.950409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **4.** Feature selection\n\nPreviously I established that logistic regression and Bernoulli Naive Bayes classifiers had best cross-validation log-loss scores (https://www.kaggle.com/code/uzdavinys/xg-expected-goals-with-simple-sklearn-models). Cross-valdiation technique will be used to evaluate impornace of having more information about historical games. Procedure:\n\n    1) select idx there all data is known for all 9 feature groups,\n    2) create 5-fold cross validation segments,\n    3) run simulations for different combiantions of features.\n    4) compare results.","metadata":{}},{"cell_type":"code","source":"# column names to count per each match\ncols = [\"_team_history_match_date_\", \"_team_history_is_play_home_\", \"_team_history_is_cup_\",\n        \"_team_history_goal_\", \"_team_history_opponent_goal_\",\n        \"_team_history_rating_\", \"_team_history_opponent_rating_\",\n        \"_team_history_coach_\", \"_team_history_league_id_\"]\n\n# summarize NaN values\ndf_train_nan = count_NaN_values_per_match(df_train_nan_all, cols)\n\n# indexes to select training features\nidxs_train = df_train_nan.NaN_min_cnt == 10\n\n# data for training and target\nX_validation = df_train.loc[idxs_train]\ny_validation = pd.DataFrame()\n\n# one-hot-encode target\nfor side in ['home', 'away', 'draw']:\n    y_validation[side] = (X_validation.target == side).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:52.952801Z","iopub.execute_input":"2022-04-29T14:58:52.953072Z","iopub.status.idle":"2022-04-29T14:58:53.184837Z","shell.execute_reply.started":"2022-04-29T14:58:52.953039Z","shell.execute_reply":"2022-04-29T14:58:53.183830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create K-valdiation\ncv_split = KFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:53.186451Z","iopub.execute_input":"2022-04-29T14:58:53.186695Z","iopub.status.idle":"2022-04-29T14:58:53.191401Z","shell.execute_reply.started":"2022-04-29T14:58:53.186664Z","shell.execute_reply":"2022-04-29T14:58:53.190474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **4.1** First attempt\n\nInvestigate how number of goals and time since last game effect cv- scores using symetrical approach: last n number of matches for both home and away teams.\n\n* `goals` simply return number of goals team and opponennt scored in last `n` game,\n* `date` day difference since `n` match for home and away teams,\n* `goal diff` calculate goal difference for historical games.","metadata":{}},{"cell_type":"code","source":"def run_cv(X_validation, y_validation, feat_cols):\n    \"\"\"\n    This function runs cross-validation of selected features and n number of last games using logistic regression.\n    \n    :param X_validation: pandas DataFrame\n    :param y_validation: pandas DataFrame\n    :param feat_cols: dict, column names for difference scenarios\n        \n    :return: list (will be used to create pandas DataFrame)\n    \"\"\" \n    \n    # tmp. list to store K-fold validation results\n    _data = list()\n\n    # iterate over scenarios\n    for feat_names in feat_cols.keys():\n        # start recording time\n        _sf = time.time()\n\n        # select columns\n        col_names = feat_cols[feat_names]\n\n        # select features for training featues\n        _X_validation = select_features(X_validation, col_names)\n\n        # count number of features per game\n        feat_cnt = (_X_validation.shape[1] - 1) // 10\n\n        # use robust scaller to transform selected features\n        scaller = RobustScaler()\n\n        # fit and transform selected train dataset sample\n        _X_validation_s = scaller.fit_transform(_X_validation)\n\n        # iterate over n (the number of histrical games)\n        for n in range(1, 11):\n            # create new log. reg. model (limit max. iteration count to 200 to save time if solution doesn't converge)\n            clf = OneVsRestClassifier(LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=200))\n\n            # start recording time\n            _sn = time.time()\n\n            # k-fold number\n            i = 1\n\n            # select n number for columns\n            __X_train_s = _X_validation_s[:, :1+(n*feat_cnt)]\n\n            # iterate over K-fold splits\n            for train_index, valid_index in cv_split.split(_X_validation_s):\n\n                # select tmp. data for training and validation\n                _X_train, _X_valid = __X_train_s[train_index], __X_train_s[valid_index]\n                _y_train, _y_valid = y_validation.values[train_index], y_validation.values[valid_index]\n\n                # train model\n                clf.fit(_X_train, _y_train)\n\n                # make predictions on train and validation data\n                _y_train_pred = clf.predict_proba(_X_train)\n                _y_valid_pred = clf.predict_proba(_X_valid)\n\n                # calculate log-loss scores\n                _score_train = log_loss(_y_train, _y_train_pred)\n                _score_valid = log_loss(_y_valid, _y_valid_pred)\n\n                # add results\n                _data.append([feat_names, n, i, _score_train, _score_valid])\n\n                # add count\n                i += 1\n\n            # print status\n            print(f\"{feat_names} tested on n={n} away historical matches in {(time.time() - _sn):.1f} sec.\")\n        print(f\"[{feat_names}] features tested in {(time.time() - _sf):.1f} sec.\")\n        \n    # return cv-scores\n    return _data","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:53.193302Z","iopub.execute_input":"2022-04-29T14:58:53.193600Z","iopub.status.idle":"2022-04-29T14:58:53.209279Z","shell.execute_reply.started":"2022-04-29T14:58:53.193569Z","shell.execute_reply":"2022-04-29T14:58:53.208016Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# column names for difference scenarios\nfeat_cols = {\"goals\": [\"goals\"],\n             \"goal diff\": [\"goal diff\"],\n             \"goals + date\": [\"goals\", 'date'],\n             \"goal diff + date\": [\"goal diff\", 'date'],\n             \"goals + goal diff\": [\"goals\", 'goal diff']}\n\n# run CV for selected scenarions\n_data = run_cv(X_validation, y_validation, feat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T14:58:53.210817Z","iopub.execute_input":"2022-04-29T14:58:53.211674Z","iopub.status.idle":"2022-04-29T15:16:00.200822Z","shell.execute_reply.started":"2022-04-29T14:58:53.211623Z","shell.execute_reply":"2022-04-29T15:16:00.199076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create DataFrame to store cross-validation results\ndf_kf_scores = pd.DataFrame(_data, columns=['Feat names', 'n games', 'k_fold', 'Train score', 'Validation score'])\n\n# quick visualization\nfig = px.box(df_kf_scores, x=\"n games\", y=\"Validation score\", color=\"Feat names\")\nfig","metadata":{"execution":{"iopub.status.busy":"2022-04-29T15:16:00.209352Z","iopub.execute_input":"2022-04-29T15:16:00.212764Z","iopub.status.idle":"2022-04-29T15:16:00.376735Z","shell.execute_reply.started":"2022-04-29T15:16:00.212671Z","shell.execute_reply":"2022-04-29T15:16:00.375826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **4.2** Second attempt\n\nFrom first attempt it seems that number of goals and date (day difference between last games) provides most robust CV scores. Let's investigate worst case scenario, no information is available for one of the teams:\n\n* `just away games` we only known `n` number of historical data for away team (goals scored and date),\n* `just home games` we only known `n` number of historical data for home team.","metadata":{}},{"cell_type":"code","source":"# tmp. list to store K-fold validation results\n_data = list()\n\n# column names for difference scenarios\nfeat_cols = {\"just away games: goals + date\": [\"goals\", 'date'],\n             \"just home games: goals + date\": [\"goals\", 'date']}\n\n# iterate over scenarios\nfor feat_names in feat_cols.keys():\n    # start recording time\n    _sf = time.time()\n    \n    # select columns\n    col_names = feat_cols[feat_names]\n    \n    # select features for training featues\n    _X_validation = select_features(X_validation, col_names)\n    \n    # select only away games info\n    if feat_names == 'just away games: goals + date':\n        # get 1 game from home\n        cols = ['home_days_1', 'home_team_history_goal_1', 'home_team_history_opponent_goal_1']\n        # add away games\n        cols += [_ for _ in _X_validation.columns[1:] if 'away' in _]\n    else:\n        # get 1 game from away\n        cols = ['away_days_1', 'away_team_history_goal_1', 'away_team_history_opponent_goal_1']\n        # add home games\n        cols += [_ for _ in _X_validation.columns[1:] if 'home' in _]\n        \n    # select columns\n    _X_validation = _X_validation[['is_cup'] + cols]\n    \n    # count number of features per game\n    feat_cnt = (_X_validation.shape[1] - 4) // 10\n    \n    # use robust scaller to transform selected features\n    scaller = RobustScaler()\n    \n    # fit and transform selected train dataset sample\n    _X_validation_s = scaller.fit_transform(_X_validation)\n    \n    # iterate over n (the number of histrical games)\n    for n in range(1, 11):\n        # create new log. reg. model (limit max. iteration count to 200 to save time if solution doesn't converge)\n        clf = OneVsRestClassifier(LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=200))\n        \n        # start recording time\n        _sn = time.time()\n                \n        # k-fold number\n        i = 1\n        \n        # select n number for columns\n        __X_train_s = _X_validation_s[:, :4+(n*feat_cnt)]\n        \n        # iterate over K-fold splits\n        for train_index, valid_index in cv_split.split(_X_validation_s):\n            \n            # select tmp. data for training and validation\n            _X_train, _X_valid = __X_train_s[train_index], __X_train_s[valid_index]\n            _y_train, _y_valid = y_validation.values[train_index], y_validation.values[valid_index]\n            \n            # train model\n            clf.fit(_X_train, _y_train)\n            \n            # make predictions on train and validation data\n            _y_train_pred = clf.predict_proba(_X_train)\n            _y_valid_pred = clf.predict_proba(_X_valid)\n\n            # calculate log-loss scores\n            _score_train = log_loss(_y_train, _y_train_pred)\n            _score_valid = log_loss(_y_valid, _y_valid_pred)\n            \n            # add results\n            _data.append([feat_names, n, i, _score_train, _score_valid])\n\n            # add count\n            i += 1\n            \n        # print status\n        print(f\"{feat_names} tested on n={n} away historical matches in {(time.time() - _sn):.1f} sec.\")\n    print(f\"[{feat_names}] features tested in {(time.time() - _sf):.1f} sec.\")","metadata":{"execution":{"iopub.status.busy":"2022-04-29T15:16:00.378077Z","iopub.execute_input":"2022-04-29T15:16:00.378302Z","iopub.status.idle":"2022-04-29T15:23:51.814512Z","shell.execute_reply.started":"2022-04-29T15:16:00.378276Z","shell.execute_reply":"2022-04-29T15:23:51.813492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create DataFrame to store cross-validation results\ndf_kf_tmp = pd.DataFrame(_data, columns=['Feat names', 'n games', 'k_fold', 'Train score', 'Validation score'])\n\n# quick visualization\nfig = px.box(df_kf_tmp, x=\"n games\", y=\"Validation score\", color=\"Feat names\")\nfig","metadata":{"execution":{"iopub.status.busy":"2022-04-29T15:23:51.816349Z","iopub.execute_input":"2022-04-29T15:23:51.816887Z","iopub.status.idle":"2022-04-29T15:23:51.918541Z","shell.execute_reply.started":"2022-04-29T15:23:51.816840Z","shell.execute_reply":"2022-04-29T15:23:51.917690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add cv-scores\ndf_kf_scores = pd.concat([df_kf_scores, df_kf_tmp]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T15:23:51.920086Z","iopub.execute_input":"2022-04-29T15:23:51.920966Z","iopub.status.idle":"2022-04-29T15:23:51.927794Z","shell.execute_reply.started":"2022-04-29T15:23:51.920901Z","shell.execute_reply":"2022-04-29T15:23:51.926980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare 2 cases\ndf_kf_plot = df_kf_scores[df_kf_scores['Feat names'].isin(['just home games: goals + date', 'goals + date'])]\n# quick visualization\nfig = px.box(df_kf_plot, x=\"n games\", y=\"Validation score\", color=\"Feat names\")\nfig","metadata":{"execution":{"iopub.status.busy":"2022-04-29T15:23:51.929043Z","iopub.execute_input":"2022-04-29T15:23:51.929262Z","iopub.status.idle":"2022-04-29T15:23:52.008905Z","shell.execute_reply.started":"2022-04-29T15:23:51.929234Z","shell.execute_reply":"2022-04-29T15:23:52.008038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **4.3** Third attempt\n\nFrom first investigation I will use date and goals scored and then add additional features:\n\n* `match outcome` one-hot-encoded feature,\n* `at home` 1 if game was played at home else 0,\n* `same league` checks if the same league id as for current match,\n* `same coach` checks if the same coach id as for current match,\n* `is cup` boolean, 1- if league else 0,\n* `ratings`,\n* `ratings v2` modified raings.","metadata":{}},{"cell_type":"code","source":"# column names for difference scenarios\nfeat_cols = {\"1st + outcome\": [\"goals\", 'date', \"match outcome\"],\n             \"1st + at home\": [\"goals\", 'date', 'play @home'],\n             \"1st + same league\": [\"goals\", 'date', 'same league'],\n             \"1st + same coach\": [\"goals\", 'date', 'same coach'],\n             \"1st + is cup\": [\"goals\", 'date', 'is cup'],\n             \"1st + ratings\": [\"goals\", 'date', 'ratings'],\n             \"1st + ratings v2\": [\"goals\", 'date', 'ratings v2']\n             }\n\n# run CV for selected scenarions\n_data = run_cv(X_validation, y_validation, feat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T15:23:52.010212Z","iopub.execute_input":"2022-04-29T15:23:52.010519Z","iopub.status.idle":"2022-04-29T16:48:51.890473Z","shell.execute_reply.started":"2022-04-29T15:23:52.010490Z","shell.execute_reply":"2022-04-29T16:48:51.889407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create DataFrame to store cross-validation results\ndf_kf_tmp = pd.DataFrame(_data, columns=['Feat names', 'n games', 'k_fold', 'Train score', 'Validation score'])\n\n# quick visualization\nfig = px.box(df_kf_tmp, x=\"n games\", y=\"Validation score\", color=\"Feat names\")\nfig","metadata":{"execution":{"iopub.status.busy":"2022-04-29T16:48:51.892144Z","iopub.execute_input":"2022-04-29T16:48:51.892815Z","iopub.status.idle":"2022-04-29T16:48:52.012126Z","shell.execute_reply.started":"2022-04-29T16:48:51.892766Z","shell.execute_reply":"2022-04-29T16:48:52.011249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add cv-scores\ndf_kf_scores = pd.concat([df_kf_scores, df_kf_tmp]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T16:48:52.013470Z","iopub.execute_input":"2022-04-29T16:48:52.014142Z","iopub.status.idle":"2022-04-29T16:48:52.020998Z","shell.execute_reply.started":"2022-04-29T16:48:52.014094Z","shell.execute_reply":"2022-04-29T16:48:52.019984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare 3 cases\ndf_kf_plot = df_kf_scores[df_kf_scores['Feat names'].isin(['1st + outcome', 'goals + date',\n                                                           '1st + at home', '1st + is cup',\n                                                           \"1st + same league\", \"1st + same coach\"])]\n# quick visualization\nfig = px.box(df_kf_plot, x=\"n games\", y=\"Validation score\", color=\"Feat names\")\nfig","metadata":{"execution":{"iopub.status.busy":"2022-04-29T16:48:52.022479Z","iopub.execute_input":"2022-04-29T16:48:52.022717Z","iopub.status.idle":"2022-04-29T16:48:52.119654Z","shell.execute_reply.started":"2022-04-29T16:48:52.022687Z","shell.execute_reply":"2022-04-29T16:48:52.118856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare 3 cases\ndf_kf_plot = df_kf_scores[df_kf_scores['Feat names'].isin(['1st + ratings', 'goals + date'])]\n# quick visualization\nfig = px.box(df_kf_plot, x=\"n games\", y=\"Validation score\", color=\"Feat names\")\nfig","metadata":{"execution":{"iopub.status.busy":"2022-04-29T16:48:52.121207Z","iopub.execute_input":"2022-04-29T16:48:52.124749Z","iopub.status.idle":"2022-04-29T16:48:52.199375Z","shell.execute_reply.started":"2022-04-29T16:48:52.124707Z","shell.execute_reply":"2022-04-29T16:48:52.198453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **5.** Train best model\n\nSelect rows for training by identifying missing rows.","metadata":{}},{"cell_type":"code","source":"# best columns to select\ncols = [\"_team_history_match_date_\", \"_team_history_goal_\", \"_team_history_opponent_goal_\",\n        \"_team_history_rating_\", \"_team_history_opponent_rating_\", \"_team_history_league_id_\", \"_team_history_is_cup_\"]\n\n# get whatever n-th game has required colun values\ndf_train_nan = count_NaN_values_per_match(df_train_nan_all, cols)\ndf_test_nan = count_NaN_values_per_match(df_test_nan_all, cols)\n\n# recount known last n matches streak\nfor df in [df_train_nan, df_test_nan]:\n    # home games\n    df.iloc[:, 0:20:2] = df.iloc[:, 0:20:2].cumsum(axis=1)\n    # away games\n    df.iloc[:, 1:20:2] = df.iloc[:, 1:20:2].cumsum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T13:18:52.903618Z","iopub.execute_input":"2022-04-28T13:18:52.903902Z","iopub.status.idle":"2022-04-28T13:18:53.215756Z","shell.execute_reply.started":"2022-04-28T13:18:52.903875Z","shell.execute_reply":"2022-04-28T13:18:53.214818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **5.1** Symmetrical predictions\n\nMake sure that same number of historical matches are available for both home and away teams.\n\n**THIS part is commented is assymetrical predictions are used**\n\n* Symmetrical data training is always faster,\n* I ran these cells for benchmarking same features symmtetrical LB score vs assymmetrical.\n\nStart from naive predictions- for games there no meaningfull information is available.","metadata":{}},{"cell_type":"code","source":"# make first prediction using average values for games during cup and not in cup\ndf_probs = pd.DataFrame(df_train.groupby('is_cup')['target'].value_counts() / df_train.groupby('is_cup')['target'].count()).T\ndf_probs","metadata":{"execution":{"iopub.status.busy":"2022-04-28T13:45:39.203538Z","iopub.execute_input":"2022-04-28T13:45:39.203813Z","iopub.status.idle":"2022-04-28T13:45:39.242165Z","shell.execute_reply.started":"2022-04-28T13:45:39.203784Z","shell.execute_reply":"2022-04-28T13:45:39.241291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create DataFrame to store predictions\ny_pred = pd.DataFrame(df_test.id)\n\ncond = (df_test_nan.NaN_min_cnt >= 0)\n\n# add long-term probabilities aquired from train dataset\ny_pred.loc[cond & (df_test.is_cup==0), 'home_s_0'] = 0.434118\ny_pred.loc[cond & (df_test.is_cup==0), 'away_s_0'] = 0.313498\ny_pred.loc[cond & (df_test.is_cup==0), 'draw_s_0'] = 0.252384\n\n# same just for games played in cup\ny_pred.loc[cond & (df_test.is_cup==1), 'home_s_0'] = 0.428918\ny_pred.loc[cond & (df_test.is_cup==1), 'away_s_0'] = 0.357009\ny_pred.loc[cond & (df_test.is_cup==1), 'draw_s_0'] = 0.214073\n\ny_pred.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T13:45:39.461852Z","iopub.execute_input":"2022-04-28T13:45:39.462516Z","iopub.status.idle":"2022-04-28T13:45:39.487071Z","shell.execute_reply.started":"2022-04-28T13:45:39.46248Z","shell.execute_reply":"2022-04-28T13:45:39.486462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best feat. names\ncol_names = [\"goals\", 'date', 'ratings', 'same league', 'is cup']\n\n# iterate over n (the number of histrical games)\nfor n in range(1, 11):\n    # start recording time\n    s = time.time()\n    \n    # create classifier\n    cls_best = OneVsRestClassifier(LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=1000))\n    \n    # select indexes for train and test datasets\n    _index_train = (df_train_nan[f\"NaN_home_team_{n}\"] == n) & (df_train_nan[f\"NaN_away_team_{n}\"] == n)\n    _index_test = (df_test_nan[f\"NaN_home_team_{n}\"] == n) & (df_test_nan[f\"NaN_away_team_{n}\"] == n)\n    \n    # select train and test input data which have symetrical ammount of historical data\n    _X_train = df_train.loc[_index_train].copy()\n    _X_test = df_test.loc[_index_test].copy()\n    \n    # use same indexes to select train target rows\n    _y_train = y_train.loc[_index_train].copy()\n    \n    # select features\n    _X_train = select_features(_X_train, col_names)\n    _X_test = select_features(_X_test, col_names)\n    \n    # count number of features per game\n    feat_cnt = (_X_train.shape[1] - 1) // 10\n    \n    # select n- number of historic games\n    _X_train = _X_train.iloc[:, :1+(n*feat_cnt)]\n    _X_test = _X_test.iloc[:, :1+(n*feat_cnt)] \n    \n    # create new robust scaller\n    scaller = RobustScaler()\n    # fit on train dataset\n    scaller.fit(_X_train)\n    \n    # transform both train and test datasets\n    _X_train_s = scaller.transform(_X_train)\n    _X_test_s = scaller.transform(_X_test)\n    \n    # train model\n    cls_best.fit(_X_train_s, _y_train)\n    # make predictions\n    _pred = cls_best.predict_proba(_X_test_s)\n    \n    # add predictions for each outcome\n    for i, side in enumerate(['home', 'away', 'draw']):\n        # save prediction\n        y_pred.loc[_index_test, f\"{side}_s_{n}\"] = _pred[:, i]\n    \n    # print status\n    print(f\"Trained on n={n} historical matches in {(time.time() - s):.1f} sec.\")","metadata":{"execution":{"iopub.status.busy":"2022-04-28T13:45:39.975896Z","iopub.execute_input":"2022-04-28T13:45:39.976304Z","iopub.status.idle":"2022-04-28T13:55:55.839565Z","shell.execute_reply.started":"2022-04-28T13:45:39.976269Z","shell.execute_reply":"2022-04-28T13:55:55.83854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# choose predictions with highest amount of information\nfor n in range(10, -1, -1):\n    # select matches with \n    cond = y_pred[f'home_s_{n}'].notna()\n    # drop values for predictions to the left\n    for i in range(n-1, -1, -1):\n        for side in ['home', 'draw', 'away']:\n            y_pred.loc[cond, f\"{side}_s_{i}\"] = np.nan","metadata":{"execution":{"iopub.status.busy":"2022-04-28T14:10:37.169637Z","iopub.execute_input":"2022-04-28T14:10:37.169949Z","iopub.status.idle":"2022-04-28T14:10:37.299111Z","shell.execute_reply.started":"2022-04-28T14:10:37.169918Z","shell.execute_reply":"2022-04-28T14:10:37.298419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sanity check\nplot_nan_cnt_matrix(df_test_nan.iloc[:, -13:], 'Test dataset')","metadata":{"execution":{"iopub.status.busy":"2022-04-28T14:10:38.403193Z","iopub.execute_input":"2022-04-28T14:10:38.403678Z","iopub.status.idle":"2022-04-28T14:10:38.536603Z","shell.execute_reply.started":"2022-04-28T14:10:38.403639Z","shell.execute_reply":"2022-04-28T14:10:38.535698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sanity check\n_ = y_pred.iloc[:, 1::3]\n_df = pd.DataFrame(_.notna().sum()).reset_index()\n_df['home'] = _df['index'].apply(lambda x: x.split(\"_\")[-1])\n_df['away'] = _df['index'].apply(lambda x: x.split(\"_\")[-1])\n_df = _df.pivot(index='home', columns='away', values=0).fillna(0).astype(int)\n_df = _df[[str(_) for _ in range(11)]]\n_df = _df.loc[[str(_) for _ in range(11)], :]\n\n_df.loc[[str(_) for _ in range(11)][::-1], :]\n_df","metadata":{"execution":{"iopub.status.busy":"2022-04-28T14:10:38.970177Z","iopub.execute_input":"2022-04-28T14:10:38.970903Z","iopub.status.idle":"2022-04-28T14:10:39.002124Z","shell.execute_reply.started":"2022-04-28T14:10:38.970862Z","shell.execute_reply":"2022-04-28T14:10:39.000766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# overide predictions if more information was available\nfor n in range(11):\n    y_pred.loc[~y_pred[f'home_s_{n}'].isna(), ['home', 'away', 'draw']] = y_pred.loc[~y_pred[f'home_s_{n}'].isna(), [f'home_s_{n}', f'away_s_{n}', f'draw_s_{n}']].values\n    \ny_pred.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T13:45:22.173852Z","iopub.execute_input":"2022-04-28T13:45:22.174145Z","iopub.status.idle":"2022-04-28T13:45:22.340908Z","shell.execute_reply.started":"2022-04-28T13:45:22.174114Z","shell.execute_reply":"2022-04-28T13:45:22.340054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# final check before saving\ny_pred[['id', 'home', 'away', 'draw']].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T13:29:56.418816Z","iopub.execute_input":"2022-04-28T13:29:56.419112Z","iopub.status.idle":"2022-04-28T13:29:56.431594Z","shell.execute_reply.started":"2022-04-28T13:29:56.419069Z","shell.execute_reply":"2022-04-28T13:29:56.430375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# uncomment if you want single predictions (just symetrical)\n\n# make predictions\nsubmission = y_pred[['id', 'home', 'away', 'draw']]\n# save predictions\nsubmission[['id', 'home', 'away', 'draw']].to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T13:29:56.432946Z","iopub.execute_input":"2022-04-28T13:29:56.43321Z","iopub.status.idle":"2022-04-28T13:29:56.819452Z","shell.execute_reply.started":"2022-04-28T13:29:56.433178Z","shell.execute_reply":"2022-04-28T13:29:56.818593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **4.2** Asymmetrical predictions\n\nUse maximum number of available historcial matches for home and away team.\n\nStart for naive predictions.","metadata":{}},{"cell_type":"code","source":"# create DataFrame to store predictions\ny_pred_a = pd.DataFrame(df_test.id)\n\ncond = (df_test_nan.NaN_home_cnt >= 0) & (df_test_nan.NaN_away_cnt >= 0)\n\n# add long-term probabilities aquired from train dataset\ny_pred_a.loc[cond & (df_test.is_cup==0), 'home_n_0_0'] = 0.434118\ny_pred_a.loc[cond & (df_test.is_cup==0), 'away_n_0_0'] = 0.313498\ny_pred_a.loc[cond & (df_test.is_cup==0), 'draw_n_0_0'] = 0.252384\n\n# same just for games played in cup\ny_pred_a.loc[cond & (df_test.is_cup==1), 'home_n_0_0'] = 0.428918\ny_pred_a.loc[cond & (df_test.is_cup==1), 'away_n_0_0'] = 0.357009\ny_pred_a.loc[cond & (df_test.is_cup==1), 'draw_n_0_0'] = 0.214073\n\ny_pred_a.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T06:53:36.019248Z","iopub.execute_input":"2022-04-28T06:53:36.019777Z","iopub.status.idle":"2022-04-28T06:53:36.048867Z","shell.execute_reply.started":"2022-04-28T06:53:36.019743Z","shell.execute_reply":"2022-04-28T06:53:36.048256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best feat. names\ncol_names = [\"goals\", 'date', 'ratings', ]\n\n# train with 0 values for home games\n# iterate over n (the number of histrical games)\nfor n in range(1, 11):\n    # start recording time\n    s = time.time()\n    \n    # select indexes for train and test datasets\n    _index_train = df_train_nan[f\"NaN_away_team_{n}\"] == n\n    _index_test = (df_test_nan.NaN_home_cnt == 0) & (df_test_nan.NaN_away_cnt == n)\n    \n    # create classifier\n    cls_best = OneVsRestClassifier(LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=1000))\n\n    # select features\n    _X_train = select_features(df_train.loc[_index_train], col_names)\n    _X_test = select_features(df_test.loc[_index_test], col_names)\n\n    # select column names for trainings\n    _col = ['is_cup']\n    # add away team data\n    _col += [_ for _ in _X_train.columns if \"away_\" in _ and int(_.split(\"_\")[-1]) in list(range(1, n+1))]\n    \n    # select train and test input data which have assymetrical ammount of historical data\n    _X_train = _X_train[_col].copy()\n    _X_test = _X_test[_col].copy()\n    # use same indexes to select train target rows\n    _y_train = y_train.loc[_index_train].copy()\n\n    # create new robust scaller\n    scaller = RobustScaler()\n    # fit on train dataset\n    scaller.fit(_X_train)\n\n    # transform both train and test datasets\n    _X_train_s = scaller.transform(_X_train)\n    _X_test_s = scaller.transform(_X_test)\n\n    # train model\n    cls_best.fit(_X_train_s, _y_train)\n\n    # make predictions\n    _pred = cls_best.predict_proba(_X_test_s)\n\n    # add predictions for each outcome\n    for i, side in enumerate(['home', 'away', 'draw']):\n        # save prediction\n        y_pred_a.loc[_index_test, f\"{side}_n_0_{n}\"] = _pred[:, i]\n\n    # to avoid defragmantation\n    y_pred_a = y_pred_a.copy()\n\n    # print status\n    print(f\"Trained on home=0, away={n} historical matches in {(time.time() - s):.1f} sec.\")\n\n# train with 0 values for away games\n# iterate over n (the number of histrical games)\nfor n in range(1, 11):\n    # start recording time\n    s = time.time()\n    \n    # select indexes for train and test datasets\n    _index_train = df_train_nan[f\"NaN_home_team_{n}\"] == n\n    _index_test = (df_test_nan.NaN_home_cnt == n) & (df_test_nan.NaN_away_cnt == 0)\n    \n    # create classifier\n    cls_best = OneVsRestClassifier(LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=1000))\n\n    # select features\n    _X_train = select_features(df_train.loc[_index_train], col_names)\n    _X_test = select_features(df_test.loc[_index_test], col_names)\n\n    # select column names for trainings\n    _col = ['is_cup']\n    # add away team data\n    _col += [_ for _ in _X_train.columns if \"home_\" in _ and int(_.split(\"_\")[-1]) in list(range(1, n+1))]\n    \n    # select train and test input data which have assymetrical ammount of historical data\n    _X_train = _X_train[_col].copy()\n    _X_test = _X_test[_col].copy()\n\n    # use same indexes to select train target rows\n    _y_train = y_train.loc[_index_train].copy()\n\n    # create new robust scaller\n    scaller = RobustScaler()\n    # fit on train dataset\n    scaller.fit(_X_train)\n\n    # transform both train and test datasets\n    _X_train_s = scaller.transform(_X_train)\n    _X_test_s = scaller.transform(_X_test)\n\n    # train model\n    cls_best.fit(_X_train_s, _y_train)\n\n    # make predictions\n    _pred = cls_best.predict_proba(_X_test_s)\n\n    # add predictions for each outcome\n    for i, side in enumerate(['home', 'away', 'draw']):\n        # save prediction\n        y_pred_a.loc[_index_test, f\"{side}_n_{n}_0\"] = _pred[:, i]\n\n    # to avoid defragmantation\n    y_pred_a = y_pred_a.copy()\n\n    # print status\n    print(f\"Trained on home={n}, away=0 historical matches in {(time.time() - s):.1f} sec.\")","metadata":{"execution":{"iopub.status.busy":"2022-04-28T06:54:00.153569Z","iopub.execute_input":"2022-04-28T06:54:00.153938Z","iopub.status.idle":"2022-04-28T06:57:02.836459Z","shell.execute_reply.started":"2022-04-28T06:54:00.153906Z","shell.execute_reply":"2022-04-28T06:57:02.83552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# iterate over n (the number of histrical games)\nfor n in range(1, 11):\n    \n    # iterate horizontally\n    for nn in range(n, 11):\n        # start recording time\n        s = time.time()\n        \n        # select indexes for train and test datasets\n        _index_train = (df_train_nan[f\"NaN_home_team_{nn}\"] == nn) & (df_train_nan[f\"NaN_away_team_{n}\"] == n)\n        _index_test = (df_test_nan[f\"NaN_home_team_{nn}\"] == nn) & (df_test_nan[f\"NaN_away_team_{n}\"] == n)\n        \n        # create classifier\n        cls_best = OneVsRestClassifier(LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=1000))\n        \n        # select features\n        _X_train = select_features(df_train.loc[_index_train], col_names)\n        _X_test = select_features(df_test.loc[_index_test], col_names)\n        \n        # select column names for trainings\n        _col = ['is_cup']\n        # add away team data\n        _col += [_ for _ in _X_train.columns if \"away_\" in _ and int(_.split(\"_\")[-1]) in list(range(1, n+1))]\n        # add home team data\n        _col += [_ for _ in _X_train.columns if \"home_\" in _ and int(_.split(\"_\")[-1]) in list(range(1, nn+1))]\n        \n        # select train and test input data which have assymetrical ammount of historical data\n        _X_train = _X_train[_col].copy()\n        _X_test = _X_test[_col].copy()\n        \n        # use same indexes to select train target rows\n        _y_train = y_train.loc[_index_train].copy()\n        \n        # create new robust scaller\n        scaller = RobustScaler()\n        # fit on train dataset\n        scaller.fit(_X_train)\n\n        # transform both train and test datasets\n        _X_train_s = scaller.transform(_X_train)\n        _X_test_s = scaller.transform(_X_test)\n\n        # train model\n        cls_best.fit(_X_train_s, _y_train)\n        \n        # make predictions\n        _pred = cls_best.predict_proba(_X_test_s)\n\n        # add predictions for each outcome\n        for i, side in enumerate(['home', 'away', 'draw']):\n            # save prediction\n            y_pred_a.loc[_index_test, f\"{side}_n_{nn}_{n}\"] = _pred[:, i]\n            \n        # to avoid defragmantation\n        y_pred_a = y_pred_a.copy()\n        \n        # print status\n        print(f\"Trained on home={nn}, away={n} historical matches in {(time.time() - s):.1f} sec.\")\n        \n    # iterate vertically\n    for nn in range(n, 10):\n        # start recording time\n        s = time.time()\n        \n        # select indexes for train and test datasets\n        _index_train = (df_train_nan[f\"NaN_home_team_{n}\"] == n) & (df_train_nan[f\"NaN_away_team_{nn+1}\"] == nn+1)\n        _index_test = (df_test_nan[f\"NaN_home_team_{n}\"] == n) & (df_test_nan[f\"NaN_away_team_{nn+1}\"] == nn+1)\n        \n        # create classifier\n        cls_best = OneVsRestClassifier(LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=1000))\n        \n        # select features\n        _X_train = select_features(df_train.loc[_index_train], col_names)\n        _X_test = select_features(df_test.loc[_index_test], col_names)\n        \n        # select column names for trainings\n        _col = ['is_cup']\n        # add home team data\n        _col += [_ for _ in _X_train.columns if \"home_\" in _ and int(_.split(\"_\")[-1]) in list(range(1, n+1))]\n        # add away team data\n        _col += [_ for _ in _X_train.columns if \"away_\" in _ and int(_.split(\"_\")[-1]) in list(range(1, nn+2))]\n        \n        # select train and test input data which have assymetrical ammount of historical data\n        _X_train = _X_train[_col].copy()\n        _X_test = _X_test[_col].copy()\n        # use same indexes to select train target rows\n        _y_train = y_train.loc[_index_train].copy()\n        \n        # create new robust scaller\n        scaller = RobustScaler()\n        # fit on train dataset\n        scaller.fit(_X_train)\n\n        # transform both train and test datasets\n        _X_train_s = scaller.transform(_X_train)\n        _X_test_s = scaller.transform(_X_test)\n\n        # train model\n        cls_best.fit(_X_train_s, _y_train)\n        # make predictions\n        _pred = cls_best.predict_proba(_X_test_s)\n\n        # add predictions for each outcome\n        for i, side in enumerate(['home', 'away', 'draw']):\n            # save prediction\n            y_pred_a.loc[_index_test, f\"{side}_n_{n}_{nn+1}\"] = _pred[:, i]\n            \n        # to avoid defragmantation\n        y_pred_a = y_pred_a.copy()\n        \n        # print status\n        print(f\"Trained on home={n}, away={nn+1} historical matches in {(time.time() - s):.1f} sec.\")","metadata":{"execution":{"iopub.status.busy":"2022-04-28T07:26:44.912829Z","iopub.execute_input":"2022-04-28T07:26:44.913546Z","iopub.status.idle":"2022-04-28T08:10:09.212599Z","shell.execute_reply.started":"2022-04-28T07:26:44.913505Z","shell.execute_reply":"2022-04-28T08:10:09.211124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# keep predictions with highest amount of information\nfor i in range(len(y_pred_a.columns)-3, 1, -3):\n    cond = y_pred_a.iloc[:, i].notna()\n    cond = y_pred_a.loc[cond].index\n    y_pred_a.iloc[cond, 1:i] = np.nan","metadata":{"execution":{"iopub.status.busy":"2022-04-28T08:11:40.446888Z","iopub.execute_input":"2022-04-28T08:11:40.447218Z","iopub.status.idle":"2022-04-28T08:11:49.73611Z","shell.execute_reply.started":"2022-04-28T08:11:40.447184Z","shell.execute_reply":"2022-04-28T08:11:49.735222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sanity check\nplot_nan_cnt_matrix(df_test_nan.iloc[:, -13:], 'Test dataset, Best case scenario')","metadata":{"execution":{"iopub.status.busy":"2022-04-28T08:11:49.737793Z","iopub.execute_input":"2022-04-28T08:11:49.738271Z","iopub.status.idle":"2022-04-28T08:11:49.923036Z","shell.execute_reply.started":"2022-04-28T08:11:49.738231Z","shell.execute_reply":"2022-04-28T08:11:49.922213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sanity check\n_ = y_pred_a.iloc[:, 1::3]\n_df = pd.DataFrame(_.notna().sum()).reset_index()\n_df['home'] = _df['index'].apply(lambda x: x.split(\"_\")[-2])\n_df['away'] = _df['index'].apply(lambda x: x.split(\"_\")[-1])\n_df = _df.pivot(index='home', columns='away', values=0)\n_df = _df[[str(_) for _ in range(11)]]\n_df = _df.loc[[str(_) for _ in range(11)], :]\n\n_df.loc[[str(_) for _ in range(11)][::-1], :]","metadata":{"execution":{"iopub.status.busy":"2022-04-28T08:11:49.924341Z","iopub.execute_input":"2022-04-28T08:11:49.924573Z","iopub.status.idle":"2022-04-28T08:11:49.991723Z","shell.execute_reply.started":"2022-04-28T08:11:49.924545Z","shell.execute_reply":"2022-04-28T08:11:49.990761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# overide predictions if more information was available\nfor x in range(11):\n    for y in range(x, 11):\n        \n        # select condition\n        _cond = y_pred_a[f'home_n_{x}_{y}'].isna()\n        y_pred_a.loc[~_cond, ['home', 'away', 'draw']] = y_pred_a.loc[~_cond, [f'home_n_{x}_{y}', f'away_n_{x}_{y}', f'draw_n_{x}_{y}']].values\n        \n        if y != x:\n            # select condition\n            _cond = y_pred_a[f'home_n_{y}_{x}'].isna()\n            y_pred_a.loc[~_cond, ['home', 'away', 'draw']] = y_pred_a.loc[~_cond, [f'home_n_{y}_{x}', f'away_n_{y}_{x}', f'draw_n_{y}_{x}']].values  \n            \ny_pred_a.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T08:11:49.993519Z","iopub.execute_input":"2022-04-28T08:11:49.993788Z","iopub.status.idle":"2022-04-28T08:11:50.842656Z","shell.execute_reply.started":"2022-04-28T08:11:49.993748Z","shell.execute_reply":"2022-04-28T08:11:50.84208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# final check before saving\ny_pred_a[['id', 'home', 'away', 'draw']].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T08:11:50.843744Z","iopub.execute_input":"2022-04-28T08:11:50.844067Z","iopub.status.idle":"2022-04-28T08:11:50.853171Z","shell.execute_reply.started":"2022-04-28T08:11:50.84404Z","shell.execute_reply":"2022-04-28T08:11:50.852555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# uncoment last cell to make submission using assymetrical trainings\n\n# # make predictions\n# submission = y_pred_a[['id', 'home', 'away', 'draw']]\n# # save predictions\n# submission[['id', 'home', 'away', 'draw']].to_csv('submission.csv', index=False)\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T08:11:55.021097Z","iopub.execute_input":"2022-04-28T08:11:55.021529Z","iopub.status.idle":"2022-04-28T08:11:55.586488Z","shell.execute_reply.started":"2022-04-28T08:11:55.0215Z","shell.execute_reply":"2022-04-28T08:11:55.585623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}