{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# linear algebra\nimport numpy as np\n# data processing\nimport pandas as pd \n# for data visualziation\nimport matplotlib.pyplot as plt\n\n# for scalling features and label encoding\nfrom sklearn.preprocessing import MinMaxScaler, LabelEncoder\n# main metric for copetition\nfrom sklearn.metrics import log_loss\n\n# bayes models\nfrom sklearn.naive_bayes import GaussianNB, BernoulliNB\n# linear models\nfrom sklearn.linear_model import LogisticRegression\n# tree classifier\nfrom sklearn.tree import DecisionTreeClassifier\n# for clusstering leagues\nfrom sklearn.cluster import KMeans\n\n# for K-Folds cross-validation\nfrom sklearn.model_selection import KFold\n\n# contants\nRANDOM_STATE = 42\n\n# plot style\nplt.style.use('ggplot')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-10T11:57:37.589245Z","iopub.execute_input":"2022-04-10T11:57:37.589569Z","iopub.status.idle":"2022-04-10T11:57:38.844562Z","shell.execute_reply.started":"2022-04-10T11:57:37.589485Z","shell.execute_reply":"2022-04-10T11:57:38.843907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The **main idea** was to develop simple sklearn model to match similiar results to LSTM model's results. I was inpisred by Vincent Warmerdam's presentation on simple models: https://www.youtube.com/watch?v=68ABAU_V8qI.\n\nTL;DR:\n\n**Feature engineering**\n\n* counted how many times home, away and their opponents scored `n=0`, `n=1`, ... and `n>=10` goals within last 10 games. In addition I counted for how many of last 10 matches information was available. Average goals (i.e. expected goals- xG) scored withing last 10 games. For matches had where no historical data, xG was set to 0.\n* counted number of games information was available, times played at home, average and max historical ratings.\n* using Kmeans leagues were clustered into 50 groups (using number of matches, avg. total goals). Some league ids were present only in test dataset, while others were only in train. These ids were clustered in new cluster \"unknown leagues\". Finally theese clusters were one-hot encoded.\n\n**Results**\n\n* Baseline #1 prediction (all outcomes have equal probability, i.e. 1/3) is **1.09861** (https://www.kaggle.com/code/hoangnguyen719/football-match-probability-prediction-baseline)\n* Baseline #2 prediction (using average outcome probabilities from train dataset [home: 0.434, draw: 0.249, away: 0.317]) is **1.07057** (https://www.kaggle.com/code/jbomitchell/constant-probability-baseline)\n* My logistic regression model (model type was chosen using K-fold cross-validaiton, best average log-loss score) is **1.02505**.\n* Best public LSTM **0.99661** (https://www.kaggle.com/code/seraquevence/top-5-football-prob-prediction-lstm-v02-gpu/notebook?scriptVersionId=91485205).\n* Bookmakers **0.97310**\n\n# 1. Load data","metadata":{}},{"cell_type":"code","source":"# load the data\ndf_train = pd.read_csv('/kaggle/input/football-match-probability-prediction/train.csv', low_memory = False)\ndf_test = pd.read_csv('/kaggle/input/football-match-probability-prediction/test.csv', low_memory = False)\nsubmission = pd.read_csv('/kaggle/input/football-match-probability-prediction/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:38.847090Z","iopub.execute_input":"2022-04-10T11:57:38.847331Z","iopub.status.idle":"2022-04-10T11:57:52.230393Z","shell.execute_reply.started":"2022-04-10T11:57:38.847295Z","shell.execute_reply":"2022-04-10T11:57:52.229542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, df_name, verbose=True):\n    \"\"\"\n    This function changes pandas numerical dtypes (reduces bit size if possible)\n    to reduce memory usage\n    \n    :param df: pandas DataFrame\n    :param df_name: str, name of DataFrame\n    :param verbose: bool, if True prints out message of how much memory usage was reduced\n        \n    :return:  pandas DataFrame       \n    \"\"\"\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int64)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float64)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    # calculate memory after reduction\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        # reduced memory usage in percent\n        diff_pst = 100 * (start_mem - end_mem) / start_mem\n        msg = f'{df_name} mem. usage decreased to {end_mem:5.2f} Mb ({diff_pst:.1f}% reduction)'\n        print(msg)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:52.231616Z","iopub.execute_input":"2022-04-10T11:57:52.231897Z","iopub.status.idle":"2022-04-10T11:57:52.244488Z","shell.execute_reply.started":"2022-04-10T11:57:52.231865Z","shell.execute_reply":"2022-04-10T11:57:52.243815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reduce memory usage if available\ndf_train = reduce_mem_usage(df_train, \"df_train\")\ndf_test = reduce_mem_usage(df_test, \"df_test\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:52.246167Z","iopub.execute_input":"2022-04-10T11:57:52.246496Z","iopub.status.idle":"2022-04-10T11:57:58.256923Z","shell.execute_reply.started":"2022-04-10T11:57:52.246469Z","shell.execute_reply":"2022-04-10T11:57:58.255910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Investigate NaN values\n\n## 2.1 expected goals","metadata":{}},{"cell_type":"code","source":"# get column names with \"goal\" string\ncol_names = [_ for _ in df_train.columns if \"goal\" in _]\n\n# count nan values for train and test datasets \ndf_nan = pd.merge(\n    pd.DataFrame(df_train[col_names].isna().sum(), columns=['train_cnt']),\n    pd.DataFrame(df_test[col_names].isna().sum(), columns=['test_cnt']),\n    left_index=True,\n    right_index=True\n)\n\n# calculate nan proportions\ndf_nan['train_pct'] = round(df_nan['train_cnt']*100 / len(df_train), 2)\ndf_nan['test_pct'] = round(df_nan['test_cnt']*100 / len(df_test), 2)\ndf_nan","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:58.258194Z","iopub.execute_input":"2022-04-10T11:57:58.258979Z","iopub.status.idle":"2022-04-10T11:57:58.392636Z","shell.execute_reply.started":"2022-04-10T11:57:58.258932Z","shell.execute_reply":"2022-04-10T11:57:58.391756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get column names with \"goal\" string\ncol_names = [_ for _ in df_train.columns if \"goal\" in _]\n# investige max goals\npd.DataFrame(df_train[col_names].max(), columns=['max'])","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:58.393871Z","iopub.execute_input":"2022-04-10T11:57:58.394211Z","iopub.status.idle":"2022-04-10T11:57:58.417805Z","shell.execute_reply.started":"2022-04-10T11:57:58.394170Z","shell.execute_reply":"2022-04-10T11:57:58.416992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.home_team_history_goal_1.hist(bins=20)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:58.419235Z","iopub.execute_input":"2022-04-10T11:57:58.419455Z","iopub.status.idle":"2022-04-10T11:57:58.665090Z","shell.execute_reply.started":"2022-04-10T11:57:58.419429Z","shell.execute_reply":"2022-04-10T11:57:58.664137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 match dates","metadata":{}},{"cell_type":"code","source":"# get column names with \"date\" string\ncol_names = [_ for _ in df_train.columns if \"date\" in _]\n\n# count nan values for train and test datasets \ndf_nan = pd.merge(\n    pd.DataFrame(df_train[col_names].isna().sum(), columns=['train_cnt']),\n    pd.DataFrame(df_test[col_names].isna().sum(), columns=['test_cnt']),\n    left_index=True,\n    right_index=True\n)\n\n# calculate nan proportions\ndf_nan['train_pct'] = round(df_nan['train_cnt']*100 / len(df_train), 2)\ndf_nan['test_pct'] = round(df_nan['test_cnt']*100 / len(df_test), 2)\ndf_nan","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:58.666470Z","iopub.execute_input":"2022-04-10T11:57:58.666776Z","iopub.status.idle":"2022-04-10T11:57:58.893999Z","shell.execute_reply.started":"2022-04-10T11:57:58.666741Z","shell.execute_reply":"2022-04-10T11:57:58.893180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.3 Leagues\n\n### 2.3.1 is cup","metadata":{}},{"cell_type":"code","source":"_ = df_train['is_cup'].isna().sum()\nprint(f\"{_} missing train data.\")\n\n_ = df_test['is_cup'].isna().sum()\nprint(f\"{_} missing train data.\")    ","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:58.895264Z","iopub.execute_input":"2022-04-10T11:57:58.895660Z","iopub.status.idle":"2022-04-10T11:57:58.908888Z","shell.execute_reply.started":"2022-04-10T11:57:58.895621Z","shell.execute_reply":"2022-04-10T11:57:58.907870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# majority of games were not played in cup\ndf_train['is_cup'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:58.912250Z","iopub.execute_input":"2022-04-10T11:57:58.912518Z","iopub.status.idle":"2022-04-10T11:57:58.929464Z","shell.execute_reply.started":"2022-04-10T11:57:58.912487Z","shell.execute_reply":"2022-04-10T11:57:58.928863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hence fill nan values with False\ndf_train['is_cup'] = df_train['is_cup'].fillna(False)\n# convert bool to int\ndf_train['is_cup'] = df_train['is_cup'].astype(int)\ndf_test['is_cup'] = df_test['is_cup'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:58.930610Z","iopub.execute_input":"2022-04-10T11:57:58.931030Z","iopub.status.idle":"2022-04-10T11:57:58.971915Z","shell.execute_reply.started":"2022-04-10T11:57:58.930988Z","shell.execute_reply":"2022-04-10T11:57:58.971213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3.2 group leagues","metadata":{"execution":{"iopub.status.busy":"2022-04-10T05:04:48.702669Z","iopub.execute_input":"2022-04-10T05:04:48.703169Z","iopub.status.idle":"2022-04-10T05:04:48.706679Z","shell.execute_reply.started":"2022-04-10T05:04:48.703135Z","shell.execute_reply":"2022-04-10T05:04:48.705858Z"}}},{"cell_type":"code","source":"# count missing values\n_ = df_train.league_name.isna().sum()\nprint(f\"{_} leagues names missing in train sample\")\n_ = df_test.league_name.isna().sum()\nprint(f\"{_} leagues names missing in train sample\")\n# fill unknown leagues with text\ndf_train.league_name = df_train.league_name.fillna(\"unknown\")\ndf_test.league_name = df_test.league_name.fillna(\"unknown\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:58.972967Z","iopub.execute_input":"2022-04-10T11:57:58.973306Z","iopub.status.idle":"2022-04-10T11:57:58.999448Z","shell.execute_reply.started":"2022-04-10T11:57:58.973277Z","shell.execute_reply":"2022-04-10T11:57:58.998515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# investigte whatever same leagues are in train and test datasets\nu1 = df_train.league_name.unique() \nu2 = df_test.league_name.unique() \n\n# temp. list to store data\n_data = list()\n\n# check if same \nfor u in u1:\n    if u not in u2:\n        _data.append([u, sum(df_train.league_name == u)])\n        \n_df = pd.DataFrame(_data, columns=['league', 'cnt_in_train_sample'])\n_df.sort_values(by='cnt_in_train_sample').tail(10)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:57:59.000564Z","iopub.execute_input":"2022-04-10T11:57:59.000815Z","iopub.status.idle":"2022-04-10T11:58:00.758508Z","shell.execute_reply.started":"2022-04-10T11:57:59.000763Z","shell.execute_reply":"2022-04-10T11:58:00.757689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# temp. list to store data\n_data = list()\n\n# check if same \nfor u in u2:\n    if u not in u1:\n        _data.append([u, sum(df_test.league_name == u)])\n        \n_df = pd.DataFrame(_data, columns=['league', 'cnt_in_test_sample'])\n_df.sort_values(by='cnt_in_test_sample').tail(10)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:00.759733Z","iopub.execute_input":"2022-04-10T11:58:00.760567Z","iopub.status.idle":"2022-04-10T11:58:02.095545Z","shell.execute_reply.started":"2022-04-10T11:58:00.760531Z","shell.execute_reply":"2022-04-10T11:58:02.094654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count number of games per top league\ndf_train.league_name.value_counts().head(10)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.097262Z","iopub.execute_input":"2022-04-10T11:58:02.097861Z","iopub.status.idle":"2022-04-10T11:58:02.112761Z","shell.execute_reply.started":"2022-04-10T11:58:02.097815Z","shell.execute_reply":"2022-04-10T11:58:02.111838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.league_name.value_counts().tail(10)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.114105Z","iopub.execute_input":"2022-04-10T11:58:02.114569Z","iopub.status.idle":"2022-04-10T11:58:02.129311Z","shell.execute_reply.started":"2022-04-10T11:58:02.114526Z","shell.execute_reply":"2022-04-10T11:58:02.128289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create DataFrame to store league ids\ndf_leagues = df_train[['league_name', 'league_id']]\n# join leagues from test dataset\ndf_leagues = pd.concat([df_leagues, df_test[['league_name', 'league_id']]])\ndf_leagues = df_leagues.drop_duplicates().reset_index(drop=True)\ndf_leagues.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.130545Z","iopub.execute_input":"2022-04-10T11:58:02.131128Z","iopub.status.idle":"2022-04-10T11:58:02.174816Z","shell.execute_reply.started":"2022-04-10T11:58:02.131084Z","shell.execute_reply":"2022-04-10T11:58:02.173929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check unknown leagues id\ndf_leagues[df_leagues.league_name == \"unknown\"]","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.176304Z","iopub.execute_input":"2022-04-10T11:58:02.176612Z","iopub.status.idle":"2022-04-10T11:58:02.186717Z","shell.execute_reply.started":"2022-04-10T11:58:02.176572Z","shell.execute_reply":"2022-04-10T11:58:02.186025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.league_id.isna().sum())\nprint(df_test.league_id.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.187715Z","iopub.execute_input":"2022-04-10T11:58:02.188466Z","iopub.status.idle":"2022-04-10T11:58:02.200437Z","shell.execute_reply.started":"2022-04-10T11:58:02.188430Z","shell.execute_reply":"2022-04-10T11:58:02.199559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_leagues[df_leagues.league_id == 1101]","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.201568Z","iopub.execute_input":"2022-04-10T11:58:02.201815Z","iopub.status.idle":"2022-04-10T11:58:02.213768Z","shell.execute_reply.started":"2022-04-10T11:58:02.201763Z","shell.execute_reply":"2022-04-10T11:58:02.212877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# how league names have duplicate names\ndf_leagues.groupby('league_id')['league_name'].count().sort_values().tail(10)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.214748Z","iopub.execute_input":"2022-04-10T11:58:02.215594Z","iopub.status.idle":"2022-04-10T11:58:02.230629Z","shell.execute_reply.started":"2022-04-10T11:58:02.215555Z","shell.execute_reply":"2022-04-10T11:58:02.229716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tmp. DataFrame to get information about league averages\ndf_res = pd.DataFrame()\n\n# iterate over last 10 games\nfor i in range(1, 11):\n    # get data for home team\n    _df = df_train[[f'home_team_history_goal_{i}', f'home_team_history_opponent_goal_{i}', f'home_team_history_league_id_{i}']].copy()\n    # rename columns\n    _df.columns = ['home_goals', 'away_goals', 'league_id']\n    # join DataFrames\n    df_res = pd.concat([df_res, _df])\n    \n    # get data for away team\n    _df = df_train[[f'away_team_history_goal_{i}', f'away_team_history_opponent_goal_{i}', f'away_team_history_league_id_{i}']].copy()\n    # rename columns\n    _df.columns = ['home_goals', 'away_goals', 'league_id']\n    # join DataFrames\n    df_res = pd.concat([df_res, _df])\n    \n# drop duplicates and missing values to return unique games\ndf_res = df_res.drop_duplicates().dropna().reset_index(drop=True)\n# convert ids to integer\ndf_res.league_id = df_res.league_id.astype(int)\ndf_res.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.232110Z","iopub.execute_input":"2022-04-10T11:58:02.232388Z","iopub.status.idle":"2022-04-10T11:58:02.574839Z","shell.execute_reply.started":"2022-04-10T11:58:02.232359Z","shell.execute_reply":"2022-04-10T11:58:02.573939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate total points scored in math\ndf_res['total_goals'] = df_res['home_goals'] + df_res['away_goals']\ndf_res.tail()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.575839Z","iopub.execute_input":"2022-04-10T11:58:02.576040Z","iopub.status.idle":"2022-04-10T11:58:02.588886Z","shell.execute_reply.started":"2022-04-10T11:58:02.576017Z","shell.execute_reply":"2022-04-10T11:58:02.587988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get statistics for each league id\ndf_league_stats = df_res.groupby('league_id').agg({'total_goals': ['median', 'mean', 'std', 'count']})\n# rename columns\ndf_league_stats.columns = [f\"stats_{_[1]}\" for _ in df_league_stats]\ndf_league_stats = df_league_stats.reset_index()\n# fill nan values with 0s\ndf_league_stats['stats_std'] = df_league_stats['stats_std'].fillna(0)\ndf_league_stats.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.590064Z","iopub.execute_input":"2022-04-10T11:58:02.590471Z","iopub.status.idle":"2022-04-10T11:58:02.623448Z","shell.execute_reply.started":"2022-04-10T11:58:02.590437Z","shell.execute_reply":"2022-04-10T11:58:02.622448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make sure no NaN valeus left\ndf_league_stats.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.624747Z","iopub.execute_input":"2022-04-10T11:58:02.625041Z","iopub.status.idle":"2022-04-10T11:58:02.633155Z","shell.execute_reply.started":"2022-04-10T11:58:02.625008Z","shell.execute_reply":"2022-04-10T11:58:02.632235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use min max scaller to normalize features prior to kmeans\nscaller = MinMaxScaler()\n# fit on train dataset\nscaller.fit(df_league_stats.iloc[:, 1:])\n# transform features\nX_kmeans = scaller.transform(df_league_stats.iloc[:, 1:])","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.634372Z","iopub.execute_input":"2022-04-10T11:58:02.634932Z","iopub.status.idle":"2022-04-10T11:58:02.650394Z","shell.execute_reply.started":"2022-04-10T11:58:02.634892Z","shell.execute_reply":"2022-04-10T11:58:02.649401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tmp. list to store data\n_data = list()\n\n# find optimal k means value\nfor n_kmeans in [_ for _ in range(2, 21)] + [_ for _ in range(22, 62, 2)]:\n    # create kmeans model\n    kmeans = KMeans(n_clusters=n_kmeans, random_state=RANDOM_STATE)\n    # fit model\n    kmeans.fit(X_kmeans)\n    # get score\n    _data.append([n_kmeans, kmeans.score(X_kmeans)])\n    \ndf_scores = pd.DataFrame(_data, columns=['kmeans', 'score'])\ndf_scores.tail()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T11:58:02.651554Z","iopub.execute_input":"2022-04-10T11:58:02.651967Z","iopub.status.idle":"2022-04-10T12:00:02.375467Z","shell.execute_reply.started":"2022-04-10T11:58:02.651922Z","shell.execute_reply":"2022-04-10T12:00:02.374336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize results\nfig, ax = plt.subplots(1, 1, figsize=(16, 4), sharex=True)\nax.plot(df_scores.kmeans, -1*df_scores.score, \"o-\")\nax.set_ylabel('Score')\nax.set_xlabel('Number of k-clussters');","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:02.388965Z","iopub.execute_input":"2022-04-10T12:00:02.392129Z","iopub.status.idle":"2022-04-10T12:00:02.606437Z","shell.execute_reply.started":"2022-04-10T12:00:02.392056Z","shell.execute_reply":"2022-04-10T12:00:02.605589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kmeans = KMeans(n_clusters=50, random_state=RANDOM_STATE)\n# fit model\nkmeans.fit(X_kmeans)\n# add fitted values\ndf_league_stats['kmeans_id'] = kmeans.predict(X_kmeans)\ndf_league_stats.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:02.607732Z","iopub.execute_input":"2022-04-10T12:00:02.608034Z","iopub.status.idle":"2022-04-10T12:00:06.761923Z","shell.execute_reply.started":"2022-04-10T12:00:02.607993Z","shell.execute_reply":"2022-04-10T12:00:06.759998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# apply kmean ID to all found games\ndf_ids = pd.merge(df_res, df_league_stats[['league_id', 'kmeans_id']], on='league_id', how='left')\ndf_ids = df_ids[['league_id', 'kmeans_id']].drop_duplicates().reset_index(drop=True)\ndf_ids.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:06.763210Z","iopub.execute_input":"2022-04-10T12:00:06.763425Z","iopub.status.idle":"2022-04-10T12:00:06.787626Z","shell.execute_reply.started":"2022-04-10T12:00:06.763401Z","shell.execute_reply":"2022-04-10T12:00:06.787039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check for NAN values\ndf_ids.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:06.789081Z","iopub.execute_input":"2022-04-10T12:00:06.789403Z","iopub.status.idle":"2022-04-10T12:00:06.799027Z","shell.execute_reply.started":"2022-04-10T12:00:06.789359Z","shell.execute_reply":"2022-04-10T12:00:06.798293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add fitted k means leagud ids\ndf_train = pd.merge(df_train, df_ids, on='league_id', how='left')\ndf_test = pd.merge(df_test, df_ids, on='league_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:06.800241Z","iopub.execute_input":"2022-04-10T12:00:06.800476Z","iopub.status.idle":"2022-04-10T12:00:07.345853Z","shell.execute_reply.started":"2022-04-10T12:00:06.800449Z","shell.execute_reply":"2022-04-10T12:00:07.344960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no historical data is available for these league ids in train data set\ndf_train[df_train.kmeans_id.isna()].league_id.unique()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:07.347439Z","iopub.execute_input":"2022-04-10T12:00:07.348064Z","iopub.status.idle":"2022-04-10T12:00:07.505959Z","shell.execute_reply.started":"2022-04-10T12:00:07.348012Z","shell.execute_reply":"2022-04-10T12:00:07.505070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no historical data is available for these league ids in train data set\ndf_test[df_test.kmeans_id.isna()].league_id.unique()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:07.507574Z","iopub.execute_input":"2022-04-10T12:00:07.507909Z","iopub.status.idle":"2022-04-10T12:00:07.585594Z","shell.execute_reply.started":"2022-04-10T12:00:07.507869Z","shell.execute_reply":"2022-04-10T12:00:07.584711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = df_train.kmeans_id.max() + 1\n\n# fill missing values with unique k-mean league id\ndf_train.kmeans_id.fillna(_, inplace=True)\ndf_test.kmeans_id.fillna(_, inplace=True)\n\n# convert to integers\ndf_train.kmeans_id = df_train.kmeans_id.astype(int)\ndf_test.kmeans_id = df_test.kmeans_id.astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:07.587180Z","iopub.execute_input":"2022-04-10T12:00:07.587494Z","iopub.status.idle":"2022-04-10T12:00:07.605048Z","shell.execute_reply.started":"2022-04-10T12:00:07.587453Z","shell.execute_reply":"2022-04-10T12:00:07.603827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make sure that same kmeans clussters are in both train and test datasets\n_set_train = set(df_train.kmeans_id.unique())\n_set_test = set(df_test.kmeans_id.unique())\n_set_test.difference(_set_train)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:07.606296Z","iopub.execute_input":"2022-04-10T12:00:07.606539Z","iopub.status.idle":"2022-04-10T12:00:07.613622Z","shell.execute_reply.started":"2022-04-10T12:00:07.606510Z","shell.execute_reply":"2022-04-10T12:00:07.613044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_set_train.difference(_set_test)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:07.614570Z","iopub.execute_input":"2022-04-10T12:00:07.614974Z","iopub.status.idle":"2022-04-10T12:00:07.629290Z","shell.execute_reply.started":"2022-04-10T12:00:07.614941Z","shell.execute_reply":"2022-04-10T12:00:07.628613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# because 20 kmeans id is only present in test but not train dataset, this ID will be converted to unknown ID\ndf_test.loc[df_test.kmeans_id == 42, \"kmeans_id\"] = df_train.kmeans_id.max() + 1","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:07.630675Z","iopub.execute_input":"2022-04-10T12:00:07.631310Z","iopub.status.idle":"2022-04-10T12:00:07.641447Z","shell.execute_reply.started":"2022-04-10T12:00:07.631268Z","shell.execute_reply":"2022-04-10T12:00:07.640216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. feature engineering\n\n## 3.1 expected goals","metadata":{}},{"cell_type":"code","source":"def add_features(df):\n    # get transformed features\n    X = df[['id', 'is_cup', 'kmeans_id']].copy()\n    \n    # iterate over column types\n    for col_id in ['home_team_history_goal_', 'home_team_history_opponent_goal_', 'away_team_history_goal_', 'away_team_history_opponent_goal_']:\n        # select column names\n        col_names = [_ for _ in df.columns if col_id in _]\n        \n        # how many times number of goals (n_goals) were scored\n        for n_goals in range(0, 10):\n            X.loc[:, f\"{col_id}{n_goals}_goals_scored\"] = (df[col_names] == n_goals).sum(axis=1)\n        \n        # how many times more than 10 goals were scored\n        X.loc[:, f\"{col_id}>=10_goals_scored\"] = (df[col_names] >= 10).sum(axis=1)\n        \n        # how many missing values\n        X.loc[:, f\"{col_id}is_nan\"] = df[col_names].isna().sum(axis=1)\n            \n        # calculate mean goals scored\n        X.loc[:, f\"{col_id}xG\"] = df[col_names].sum() / (10 - X.loc[:, f\"{col_id}is_nan\"])\n        \n        # fill missing values with 0s\n        X.loc[:, f\"{col_id}xG\"].fillna(0, inplace=True)\n        \n    # get information about rakings\n    for col_id in ['home_team_history_rating_', 'home_team_history_opponent_rating_', 'away_team_history_rating_', 'away_team_history_opponent_rating_']:\n        # average rating\n        X.loc[:, f\"{col_id}avg\"] = df[col_names].mean(axis=1)\n        \n        # max rating\n        X.loc[:, f\"{col_id}max\"] = df[col_names].max(axis=1)\n        \n        # fill missing values with 0s\n        X.loc[:, f\"{col_id}avg\"].fillna(0, inplace=True)\n        X.loc[:, f\"{col_id}max\"].fillna(0, inplace=True)\n        \n    # explore games at home\n    for col_id in ['home_team_history_is_play_home_', 'away_team_history_is_play_home_']:\n        # count how many last n games were played at home\n        X.loc[:, f\"{col_id}cnt\"] = df[col_names].sum(axis=1)\n        # fill missing values with 0s\n        X.loc[:, f\"{col_id}cnt\"].fillna(0, inplace=True)\n    \n    # return modified DataFrame\n    return X","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:07.643453Z","iopub.execute_input":"2022-04-10T12:00:07.643851Z","iopub.status.idle":"2022-04-10T12:00:07.656152Z","shell.execute_reply.started":"2022-04-10T12:00:07.643802Z","shell.execute_reply":"2022-04-10T12:00:07.655291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select features for transformation\nX_train = add_features(df_train)\nX_test = add_features(df_test)\n# select target for training\ny_train = df_train.target.copy()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:07.657242Z","iopub.execute_input":"2022-04-10T12:00:07.657611Z","iopub.status.idle":"2022-04-10T12:00:08.938043Z","shell.execute_reply.started":"2022-04-10T12:00:07.657582Z","shell.execute_reply":"2022-04-10T12:00:08.937007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make sure that same number of missing values are for both home team and oponent\nans_1 = (X_train['home_team_history_goal_is_nan'] != X_train['home_team_history_opponent_goal_is_nan']).sum()\nans_2 = (X_test['home_team_history_goal_is_nan'] != X_test['home_team_history_opponent_goal_is_nan']).sum()\n\nprint(f\"train: {ans_1}\")\nprint(f\"test: {ans_2}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:08.939521Z","iopub.execute_input":"2022-04-10T12:00:08.939880Z","iopub.status.idle":"2022-04-10T12:00:08.949018Z","shell.execute_reply.started":"2022-04-10T12:00:08.939834Z","shell.execute_reply":"2022-04-10T12:00:08.947881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# same information is present, remove that value\ndel X_train['home_team_history_opponent_goal_is_nan'], X_test['home_team_history_opponent_goal_is_nan']","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:08.950264Z","iopub.execute_input":"2022-04-10T12:00:08.950489Z","iopub.status.idle":"2022-04-10T12:00:08.958997Z","shell.execute_reply.started":"2022-04-10T12:00:08.950462Z","shell.execute_reply":"2022-04-10T12:00:08.958302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make sure that same number of missing values are for both away team and oponent\nans_1 = (X_train['away_team_history_goal_is_nan'] != X_train['away_team_history_opponent_goal_is_nan']).sum()\nans_2 = (X_test['away_team_history_goal_is_nan'] != X_test['away_team_history_opponent_goal_is_nan']).sum()\n\nprint(f\"train: {ans_1}\")\nprint(f\"test: {ans_2}\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:08.960217Z","iopub.execute_input":"2022-04-10T12:00:08.960997Z","iopub.status.idle":"2022-04-10T12:00:08.975380Z","shell.execute_reply.started":"2022-04-10T12:00:08.960962Z","shell.execute_reply":"2022-04-10T12:00:08.974472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# same information is present, remove that value\ndel X_train['away_team_history_opponent_goal_is_nan'], X_test['away_team_history_opponent_goal_is_nan']","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:08.976866Z","iopub.execute_input":"2022-04-10T12:00:08.977191Z","iopub.status.idle":"2022-04-10T12:00:08.986747Z","shell.execute_reply.started":"2022-04-10T12:00:08.977148Z","shell.execute_reply":"2022-04-10T12:00:08.986086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# different amount of information is available for home and away teams\n(X_test['home_team_history_goal_is_nan'] != X_test['away_team_history_goal_is_nan']).sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:08.988000Z","iopub.execute_input":"2022-04-10T12:00:08.988392Z","iopub.status.idle":"2022-04-10T12:00:08.997937Z","shell.execute_reply.started":"2022-04-10T12:00:08.988316Z","shell.execute_reply":"2022-04-10T12:00:08.997234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. feature transformation","metadata":{}},{"cell_type":"code","source":"# one-hot encode k-mean league clussters\nX_train = pd.concat([X_train, pd.get_dummies(X_train['kmeans_id'], prefix=\"k_cluster\")], axis=1)\nX_test = pd.concat([X_test, pd.get_dummies(X_test['kmeans_id'], prefix=\"k_cluster\")], axis=1)\n# delete kmeans_id columns\ndel X_train['kmeans_id'], X_test['kmeans_id']","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:08.999235Z","iopub.execute_input":"2022-04-10T12:00:08.999654Z","iopub.status.idle":"2022-04-10T12:00:09.090702Z","shell.execute_reply.started":"2022-04-10T12:00:08.999623Z","shell.execute_reply":"2022-04-10T12:00:09.090079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make sure test columns have same order\nX_test = X_test[X_train.columns]","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:09.092062Z","iopub.execute_input":"2022-04-10T12:00:09.092558Z","iopub.status.idle":"2022-04-10T12:00:09.124008Z","shell.execute_reply.started":"2022-04-10T12:00:09.092516Z","shell.execute_reply":"2022-04-10T12:00:09.123091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# same columns in both datasets\n(X_test.columns == X_train.columns).mean()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:09.125533Z","iopub.execute_input":"2022-04-10T12:00:09.127084Z","iopub.status.idle":"2022-04-10T12:00:09.133464Z","shell.execute_reply.started":"2022-04-10T12:00:09.127032Z","shell.execute_reply":"2022-04-10T12:00:09.132327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use min max scaller to normalize features\nscaller = MinMaxScaler()\n# fit on train dataset\nscaller.fit(X_train.iloc[:, 1:])\n# transform both train and test datasets (exclude first column)\nX_train_s = scaller.transform(X_train.iloc[:, 1:])\nX_test_s = scaller.transform(X_test.iloc[:, 1:])","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:09.134822Z","iopub.execute_input":"2022-04-10T12:00:09.135123Z","iopub.status.idle":"2022-04-10T12:00:09.487053Z","shell.execute_reply.started":"2022-04-10T12:00:09.135093Z","shell.execute_reply":"2022-04-10T12:00:09.486170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# OHE training target value\ny_train_encoded = pd.get_dummies(y_train)\ny_train_encoded.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:09.488826Z","iopub.execute_input":"2022-04-10T12:00:09.489171Z","iopub.status.idle":"2022-04-10T12:00:09.510515Z","shell.execute_reply.started":"2022-04-10T12:00:09.489127Z","shell.execute_reply":"2022-04-10T12:00:09.509615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Train models","metadata":{}},{"cell_type":"code","source":"# create K-valdiation\ncv_split = KFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:09.512353Z","iopub.execute_input":"2022-04-10T12:00:09.512677Z","iopub.status.idle":"2022-04-10T12:00:09.517493Z","shell.execute_reply.started":"2022-04-10T12:00:09.512635Z","shell.execute_reply":"2022-04-10T12:00:09.516470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.1 multi-class classification","metadata":{}},{"cell_type":"code","source":"# all trained models will be saved here\nmodels = {\"GaussianNB\": GaussianNB(),\n          \"BernoulliNB\": BernoulliNB(),\n          \"LogisticRegression\": LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=1000),\n          # at least 20 games found at final node and min split with 100 games\n          \"DecisionTreeClassifier\": DecisionTreeClassifier(random_state=RANDOM_STATE, min_samples_split=100, min_samples_leaf=10)}\n\n# tmp. list to store K-fold validation results\n_data = list()\n\nfor model_name in models.keys():\n    # k-fold indes\n    i = 1\n\n    # iterate over K-fold splits\n    for train_index, valid_index in cv_split.split(X_train_s):\n\n        # create classifier\n        clf =models[model_name]\n\n        # select tmp. data for training and validation\n        _X_train, _X_valid = X_train_s[train_index], X_train_s[valid_index]\n        _y_train, _y_valid = y_train.loc[train_index], y_train.loc[valid_index]\n\n        # train model\n        clf.fit(_X_train, _y_train)\n\n        # make predictions on train and validation data\n        _y_train_pred = clf.predict_proba(_X_train)\n        _y_valid_pred = clf.predict_proba(_X_valid)\n\n        # calculate log-loss scores\n        _score_train = log_loss(_y_train, _y_train_pred)\n        _score_valid = log_loss(_y_valid, _y_valid_pred)\n\n        # add results\n        _data.append([model_name, i, _score_train, _score_valid])\n\n        # add count\n        i += 1\n    \n# create tmp. DataFrame to store cross-validation results\ndf_kf = pd.DataFrame(_data, columns=['model_name', 'k_fold', 'train_score', 'valid_score'])\n\n# return average scores for each model\ndf_kf.groupby('model_name')[['train_score', 'valid_score']].mean().reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:00:09.519338Z","iopub.execute_input":"2022-04-10T12:00:09.519748Z","iopub.status.idle":"2022-04-10T12:02:53.793214Z","shell.execute_reply.started":"2022-04-10T12:00:09.519689Z","shell.execute_reply":"2022-04-10T12:02:53.792097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. train best model","metadata":{}},{"cell_type":"code","source":"# create classifier\ncls_best = LogisticRegression(n_jobs=-1, random_state=RANDOM_STATE, max_iter=1000)\n# fit on train data\ncls_best = cls_best.fit(X_train_s, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:02:53.794798Z","iopub.execute_input":"2022-04-10T12:02:53.795626Z","iopub.status.idle":"2022-04-10T12:03:28.897054Z","shell.execute_reply.started":"2022-04-10T12:02:53.795587Z","shell.execute_reply":"2022-04-10T12:03:28.896175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select correct column order\nidxs = cls_best.predict_proba(X_test_s).argmax(axis=1)\npreds = cls_best.predict(X_test_s)\ncols = pd.DataFrame(np.array([idxs, preds]).T, columns=['idx', 'side']).drop_duplicates().sort_values(by='idx')","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:03:28.898282Z","iopub.execute_input":"2022-04-10T12:03:28.898651Z","iopub.status.idle":"2022-04-10T12:03:28.999310Z","shell.execute_reply.started":"2022-04-10T12:03:28.898620Z","shell.execute_reply":"2022-04-10T12:03:28.998304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make predictions\nsubmission = pd.DataFrame(cls_best.predict_proba(X_test_s), columns=cols.side.values)\nsubmission['id'] = X_test['id']\n# save predictions\nsubmission[['id', 'home', 'away', 'draw']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T12:03:29.000776Z","iopub.execute_input":"2022-04-10T12:03:29.001439Z","iopub.status.idle":"2022-04-10T12:03:29.469450Z","shell.execute_reply.started":"2022-04-10T12:03:29.001393Z","shell.execute_reply":"2022-04-10T12:03:29.468852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}