{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Imports and Loading Data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import layers\nfrom sklearn.preprocessing import LabelEncoder, OrdinalEncoder\nimport lightgbm as lgb\nfrom sklearn import metrics, model_selection\nfrom sklearn.model_selection import KFold\nfrom sklearn.linear_model import LogisticRegression\n\ntf.random.set_seed(5913)\n\ntrain_path = '/kaggle/input/football-match-probability-prediction/train.csv'\ntest_path = '/kaggle/input/football-match-probability-prediction/test.csv'\ntrain_scores_path = '/kaggle/input/football-match-probability-prediction/train_target_and_scores.csv'","metadata":{"papermill":{"duration":41.711018,"end_time":"2022-04-19T02:28:38.976404","exception":false,"start_time":"2022-04-19T02:27:57.265386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T14:54:35.013368Z","iopub.execute_input":"2022-05-04T14:54:35.014008Z","iopub.status.idle":"2022-05-04T14:54:35.109198Z","shell.execute_reply.started":"2022-05-04T14:54:35.013957Z","shell.execute_reply":"2022-05-04T14:54:35.108288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Features","metadata":{}},{"cell_type":"code","source":"def prepare_df(df, statistical_features=False):\n\n    # Missing league = 9999\n    cols = list(df.filter(regex=\"league_id\"))\n    df.loc[:, cols] = df.loc[:, cols].fillna(9999)\n    df.loc[:, cols] = df.loc[:, cols].astype(int)\n    df.loc[:, cols] = df.loc[:, cols].astype(str)\n    df.loc[:, cols] = \"league\" + df.loc[:, cols]\n\n    # Missing is_home or is_play_home = 0\n    cols = list(df.filter(regex=\"is_cup|is_play_home\"))\n    df.loc[:, cols] = df.loc[:, cols].fillna(0)\n    df[\"is_cup\"] = df[\"is_cup\"].astype(int)\n\n    # Missing rating = 6\n    cols = list(df.filter(regex=\"rating\"))\n    df.loc[:, cols] = df.loc[:, cols].fillna(6)\n\n    # Missing goal = 0\n    cols = list(df.filter(regex=\"goal\"))\n    df.loc[:, cols] = df.loc[:, cols].fillna(0)\n\n    # Convert date columns to datetime\n    date_cols = list(df.filter(regex=\"date\"))\n    for col in date_cols:\n        df[col] = pd.to_datetime(df[col])\n\n    # new_cols is a dictionary of the new columns.\n    # I use this to avoid the \"frame.insert\" pandas warning\n    new_cols = {}\n    \n    # Friendly matches\n    friendly_ids = [\n        'league1101', 'league1082', 'league1363', 'league1703',\n        'league1565', 'league1652'\n    ]\n\n    # Is the current match friendly?\n    new_cols[\"is_friendly\"] = df.league_id.isin(friendly_ids).fillna(False).astype(int)    \n\n    for i in range(1, 11):\n        for team in [\"home\", \"away\"]:\n            # Mumber of days in relation to the current game\n            # Missing = 999\n            new_cols[f\"{team}_team_history_days_{i}\"] = (\n                (df[\"match_date\"] - df[f\"{team}_team_history_match_date_{i}\"]).dt.days\n            ).fillna(999)  \n\n            # Same Coach\n            new_cols[f\"{team}_team_history_same_coach_{i}\"] = (\n                df[f\"{team}_team_coach_id\"] == df[f'{team}_team_history_coach_{i}']\n            ).astype(int)\n\n            # Results for each history match\n            var = f\"{team}_team_history_goal_{i}\"\n            opponent_var = f\"{team}_team_history_opponent_goal_{i}\"\n            play_home_var = f\"{team}_team_history_is_play_home_{i}\"\n\n            conditions = [\n                ((df[var] > df[opponent_var]) & (df[play_home_var] == 1)),\n                ((df[var] < df[opponent_var]) & (df[play_home_var] == 1)),\n                ((df[var] == df[opponent_var]) & (df[play_home_var] == 1)),\n                ((df[var] > df[opponent_var]) & (df[play_home_var] == 0)),\n                ((df[var] < df[opponent_var]) & (df[play_home_var] == 0)),\n                ((df[var] == df[opponent_var]) & (df[play_home_var] == 0))\n            ]\n\n            choices = [\"win_home\", \"loss_home\", \"draw_home\",\n                    \"win_away\", \"loss_away\", \"draw_away\"]\n                    \n            new_cols[f\"{team}_team_history_result_{i}\"] = pd.Series(\n                np.select(conditions, choices)\n            )\n            \n            # Weighted Goals\n            # Put more importance on goals scored against strong teams and goals conceded\n            # for weak teams.\n            # goal scored\n            new_cols[f\"{team}_team_history_wgoal_{i}\"] = df[f\"home_team_history_goal_{i}\"] * \\\n                df[f\"home_team_history_opponent_rating_{i}\"]\n            # goal conceded            \n            new_cols[f\"{team}_team_history_opponent_wgoal_{i}\"] = \\\n                df[f\"home_team_history_opponent_goal_{i}\"] * \\\n                (50 - df[f\"home_team_history_opponent_rating_{i}\"])\n\n            # Win, Draw and Loss streaks\n            if i == 1:\n                new_cols[f\"{team}_team_win_streak\"] = np.repeat(0, len(df))\n                new_cols[f\"{team}_team_loss_streak\"] = np.repeat(0, len(df))\n                new_cols[f\"{team}_team_draw_streak\"] = np.repeat(0, len(df))\n\n            var = f\"{team}_team_history_goal_{i}\"\n            opponent_var = f\"{team}_team_history_opponent_goal_{i}\"\n            \n            new_cols[f\"{team}_team_win_streak\"] = np.where(\n                ((df[var] > df[opponent_var]) &\n                (new_cols[f\"{team}_team_win_streak\"] == i - 1)),\n                new_cols[f\"{team}_team_win_streak\"] + 1,\n                new_cols[f\"{team}_team_win_streak\"]\n            )\n\n            new_cols[f\"{team}_team_loss_streak\"] = np.where(\n                ((df[var] < df[opponent_var]) &\n                (new_cols[f\"{team}_team_loss_streak\"] == i - 1)),\n                new_cols[f\"{team}_team_loss_streak\"] + 1,\n                new_cols[f\"{team}_team_loss_streak\"]\n            )\n\n            new_cols[f\"{team}_team_draw_streak\"] = np.where(\n                ((df[var] == df[opponent_var]) &\n                (new_cols[f\"{team}_team_draw_streak\"] == i - 1)),\n                new_cols[f\"{team}_team_draw_streak\"] + 1,\n                new_cols[f\"{team}_team_draw_streak\"]\n            )\n\n            new_cols[f\"{team}_team_win_streak\"] = pd.Series(new_cols[f\"{team}_team_win_streak\"])\n            new_cols[f\"{team}_team_loss_streak\"] = pd.Series(new_cols[f\"{team}_team_loss_streak\"])\n            new_cols[f\"{team}_team_draw_streak\"] = pd.Series(new_cols[f\"{team}_team_draw_streak\"])\n\n\n            # Goals difference for each history match \n            var = f\"{team}_team_history_goal_{i}\"\n            opponent_var = f\"{team}_team_history_opponent_goal_{i}\"\n            new_cols[f\"{team}_team_history_goal_diff_{i}\"] = df[var] - df[opponent_var]\n\n            # Is friendly?\n            new_cols[f\"{team}_team_history_is_friendly_{i}\"] = (\n                df[f\"{team}_team_history_league_id_{i}\"]\n                    .isin(friendly_ids)\n                    .fillna(False)\n                    .astype(int)\n                )\n            \n            # Features from https://www.kaggle.com/code/seraquevence/top-5-football-prob-prediction-lstm-v01/notebook?scriptVersionId=90841834\n            # \"Modified\" ELO rating\n            new_cols[f\"{team}_team_history_elo_{i}\"] = 1 / (\n                1 + 10 ** ((df[f'{team}_team_history_opponent_rating_{i}'] - df[f'{team}_team_history_rating_{i}'])/10)\n            )\n\n            # Same league\n            new_cols[f\"{team}_team_history_same_league_{i}\"] = (\n                df[\"league_id\"] == df[f'home_team_history_league_id_{i}']\n            ).astype(int)\n\n        var = f\"home_team_history_rating_{i}\"\n        away_var = f\"away_team_history_rating_{i}\"\n        new_cols[f'history_rating_diff_{i}'] = df[var] - df[away_var] \n   \n   # Win, draw and loss counts\n    for i in [3, 5, 10]:\n        for team in [\"home\", \"away\"]:\n            new_cols[f\"{team}_team_win_count_{i}\"] = pd.Series(np.repeat(0, len(df)))\n            new_cols[f\"{team}_team_loss_count_{i}\"] = pd.Series(np.repeat(0, len(df)))\n            new_cols[f\"{team}_team_draw_count_{i}\"] = pd.Series(np.repeat(0, len(df)))\n\n            for j in range(1, i + 1):\n                new_cols[f\"{team}_team_win_count_{i}\"] += new_cols[f\"{team}_team_history_result_{j}\"].str.contains(\n            \"win\")\n                new_cols[f\"{team}_team_loss_count_{i}\"] += new_cols[f\"{team}_team_history_result_{j}\"].str.contains(\n            \"loss\")\n                new_cols[f\"{team}_team_draw_count_{i}\"] += new_cols[f\"{team}_team_history_result_{j}\"].str.contains(\n            \"draw\")   \n\n    new_cols = pd.concat(new_cols, axis=1)\n    df = pd.concat([df, new_cols], axis=1)\n\n    if statistical_features:\n        new_cols = {}\n\n        vars = [\n            \"rating\", \"opponent_rating\", \"goal\", \"opponent_goal\", \"wgoal\",\n            \"opponent_wgoal\", \"days\", \"is_cup\", \"same_league\", \"same_coach\",\n            \"elo\", \"goal_diff\"\n        ]\n\n        for var in vars:\n            for team in [\"home\", \"away\"]:\n                new_cols[f\"{team}_team_mean_{var}\"] = df.filter(\n                    regex=f\"{team}_team_history_{var}_[0-9]\").mean(axis=1)\n                new_cols[f\"{team}_team_median_{var}\"] = df.filter(\n                    regex=f\"{team}_team_history_{var}_[0-9]\").median(axis=1)\n                new_cols[f\"{team}_team_std_{var}\"] = df.filter(\n                    regex=f\"{team}_team_history_{var}_[0-9]\").std(axis=1)\n                new_cols[f\"{team}_team_max_{var}\"] = df.filter(\n                    regex=f\"{team}_team_history_{var}_[0-9]\").max(axis=1)\n                new_cols[f\"{team}_team_min_{var}\"] = df.filter(\n                    regex=f\"{team}_team_history_{var}_[0-9]\").min(axis=1)\n                new_cols[f\"{team}_team_last_{var}\"] = df.filter(\n                    regex=f\"{team}_team_history_{var}_1$\").mean(axis=1)\n\n            for fun in [\"mean\", \"median\", \"std\", \"max\", \"min\", \"last\"]:\n                new_cols[f\"diff_{fun}_{var}\"] = new_cols[f\"home_team_{fun}_{var}\"] - \\\n                    new_cols[f\"away_team_{fun}_{var}\"]\n                \n        new_cols = pd.concat(new_cols, axis=1)\n        df = pd.concat([df, new_cols], axis=1)\n        \n    # Order columns by name\n    df.columns = df.columns.str.replace(\"_([0-9]$)\", \"_0\\\\1\", regex=True)\n    df = df.reindex(sorted(df.columns), axis=1)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-05-04T13:46:03.553639Z","iopub.execute_input":"2022-05-04T13:46:03.553962Z","iopub.status.idle":"2022-05-04T13:46:03.603426Z","shell.execute_reply.started":"2022-05-04T13:46:03.553930Z","shell.execute_reply":"2022-05-04T13:46:03.602673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxilary Functions","metadata":{}},{"cell_type":"code","source":"def dataframe_to_dataset(dataframe, is_test=False, dummies=True):\n    dataframe = dataframe.copy()\n    if not is_test:\n        labels = dataframe.pop(\"target\")\n        if dummies:\n            labels = pd.get_dummies(labels)\n    hist_numerical_df = dataframe[history_numerical_features].copy()\n    static_numerical_df = dataframe[static_numerical_features].copy()\n    dataframe = dataframe.drop(numerical_features, axis=1)\n    features_dict = {'hist_num': hist_numerical_df.values}\n    features_dict.update({'static_num': static_numerical_df.values})\n    features_dict.update(dict(dataframe))\n\n    if is_test:\n        ds = tf.data.Dataset.from_tensor_slices(\n            (features_dict)\n        )\n    else:\n        ds = tf.data.Dataset.from_tensor_slices(\n            (features_dict, labels.values)\n        )\n\n    return ds\n\n# Funções para pré-processamento das features\n@tf.autograph.experimental.do_not_convert\ndef encode_numerical_features(feature, name, dataset):\n    # Create a Normalization layer for our feature\n    normalizer = layers.Normalization()\n\n    # Prepare a Dataset that only yields our feature\n    feature_ds = dataset.map(lambda x, y: x[name])\n\n    # Learn the statistics of the data\n    with tf.device(\"cpu\"):\n        normalizer.adapt(feature_ds)\n\n    # Normalize the input feature\n    encoded_feature = normalizer(feature)\n    return encoded_feature\n@tf.autograph.experimental.do_not_convert\ndef encode_categorical_feature(feature, name, dataset, vocab):\n\n    if len(vocab) <= 10:\n        lookup = layers.StringLookup(\n            vocabulary=vocab, output_mode=\"one_hot\"\n        )\n        encoded_feature = lookup(feature)\n    else:\n        lookup = layers.StringLookup(\n            vocabulary=vocab, output_mode=\"int\"\n        )\n        encoded_feature = layers.Embedding(\n            input_dim=lookup.vocabulary_size(),\n            output_dim=1\n        )(lookup(feature))\n\n    # Turn the string input into integer indices\n\n    encoded_feature = layers.Flatten()(encoded_feature)\n\n    return encoded_feature\n\ndef create_model(dataset, units, dropout, categorical_vocab, lr=0.01, mode=\"classification\"):\n    # Inputs\n    inputs = []\n    history_features = []\n    static_features = []\n\n    # Numerical input\n    input = keras.Input(shape=(len(history_numerical_features,)),\n                        name=\"hist_num\", dtype=\"float32\")\n    feature_encoded = encode_numerical_features(input, \"hist_num\", dataset)\n    inputs.append(input)\n    history_features.append(feature_encoded)\n\n    input = keras.Input(shape=(len(static_numerical_features,)),\n                        name=\"static_num\", dtype=\"float32\")\n    feature_encoded = encode_numerical_features(input, \"static_num\", dataset)\n    inputs.append(input)\n    static_features.append(feature_encoded)\n\n    for feature in categorical_features:\n        input = keras.Input(shape=(1,), name=feature, dtype=\"string\")\n        feature_encoded = encode_categorical_feature(\n            input, feature, dataset, categorical_vocab[feature]\n        )\n        inputs.append(input)\n        if feature in history_categorical_features:\n            history_features.append(feature_encoded)\n        else:\n            static_features.append(feature_encoded)\n\n    for feature in other_features:\n        input = keras.Input(shape=(1,), name=feature, dtype=\"float32\")\n        inputs.append(input)\n        if feature in history_other_features:\n            history_features.append(input)\n        else:\n            static_features.append(input)\n\n    all_history_features = layers.concatenate(history_features)\n    all_static_features = layers.concatenate(static_features)\n    x = layers.Reshape(\n        (int(all_history_features.shape[1]/10), 10))(all_history_features)\n    x = layers.Permute((2, 1))(x)\n    x = layers.Bidirectional(\n        layers.LSTM(units[0], return_sequences=True, activation=\"swish\")\n    )(x)\n    x = layers.Dropout(dropout[0])(x)\n    x = layers.Bidirectional(\n        layers.LSTM(units[1], return_sequences=True, activation=\"swish\")\n    )(x)\n    x = layers.Dropout(dropout[1])(x)\n    x = layers.Flatten()(x)\n    x = layers.concatenate([x, all_static_features])\n    x = layers.Dense(units[2], activation=\"swish\")(x)\n    x = layers.Dropout(dropout[2])(x)\n    \n    if mode == \"classification\":\n        output = layers.Dense(3, activation=\"softmax\")(x)\n    if mode == \"regression\":\n        output = layers.Dense(1)(x)\n        \n    model = keras.Model(inputs, output)\n\n    opt = keras.optimizers.Adam(learning_rate=lr)\n    \n    if mode == \"classification\":\n        model.compile(\n            optimizer=opt,\n            loss=\"categorical_crossentropy\",\n            metrics=[\"accuracy\"]\n        )\n    \n    if mode == \"regression\":\n        model.compile(\n            optimizer=opt,\n            loss=\"mean_squared_error\"\n        )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:35:59.519463Z","iopub.execute_input":"2022-05-04T14:35:59.519844Z","iopub.status.idle":"2022-05-04T14:35:59.552318Z","shell.execute_reply.started":"2022-05-04T14:35:59.519805Z","shell.execute_reply":"2022-05-04T14:35:59.551569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LSTM","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(train_path, low_memory=False)\ntrain_scores = pd.read_csv(train_scores_path, low_memory=False)\ntest = pd.read_csv(test_path, low_memory=False)\n\ngoal_home = train_scores.score.str.extract(r'([0-9]{1,})').astype(int)\ngoal_away = train_scores.score.str.extract(r'-([0-9]{1,})').astype(int)\ny = goal_home - goal_away\ny = np.squeeze(y.values)\n\ny_cat = train[\"target\"]\ny_reg = goal_home - goal_away\ny_reg = np.squeeze(y_reg.values)\n\ntrain = prepare_df(train, statistical_features=False)\ntest = prepare_df(test, statistical_features=False)\n\ncols_regex = \"target|rating|result|streak|goal|is_cup|league_id|days|is_play_home|same_coach|mean|friend|count|elo|same_league|scores\"\n\ntrain = train.filter(regex=cols_regex)\ntest = test.filter(regex=cols_regex)\n\n# Features lists\nhistory_features = list(train.filter(regex=\"history\"))\n\nnumerical_features = list(train.filter(\n    regex=\"rating|goal|streak|days|mean|max|min|last|count|elo|scores\"))\nhistory_numerical_features = [\n    x for x in numerical_features if x in history_features\n]\nstatic_numerical_features = [\n    x for x in numerical_features if x not in history_features\n]\n\ncategorical_features = list(train.filter(regex=\"league_id|result\"))\nhistory_categorical_features = [\n    x for x in categorical_features if x in history_features\n]\nstatic_categorical_features = [\n    x for x in categorical_features if x not in history_features\n]\n\nother_features = list(train.filter(\n    regex=\"^is_cup|^is_friend|history_is_cup|history_is_play_home|history_same_coach|history_is_friend|history_same_league\"))\nhistory_other_features = [x for x in other_features if x in history_features]\nstatic_other_features = [\n    x for x in other_features if x not in history_features\n]\n\n# Categorical Vocabulary\ncategorical_vocab = {}\nfor col in categorical_features:\n    train[col] = train[col].astype(str)\n    categorical_vocab[col] = sorted(train[col].unique())  ","metadata":{"execution":{"iopub.status.busy":"2022-05-04T13:46:03.637775Z","iopub.execute_input":"2022-05-04T13:46:03.638443Z","iopub.status.idle":"2022-05-04T13:46:49.791684Z","shell.execute_reply.started":"2022-05-04T13:46:03.638391Z","shell.execute_reply":"2022-05-04T13:46:49.790630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Regression","metadata":{}},{"cell_type":"code","source":"# Config\nbatch_size = 256\nlr = 0.00038742107942815035\nunits = [90, 52, 25]\ndropout = [0.12680238931539686, 0.10338726492821518, 0.16028874910200375]\n\n# Change target to home_goal - away_goal\ntrain[\"target\"] = y_reg\n\n# Kfold cross validation\nkf = KFold(n_splits=10, shuffle=True, random_state=32)\nkf.get_n_splits(train)\noof_pred = pd.DataFrame()\nscores = []\n\n# Test\ntest_ds = dataframe_to_dataset(test, is_test=True)\ntest_ds = test_ds.batch(batch_size)\npreds = []\n\nfor train_index, val_index in kf.split(train):\n    trn = train.iloc[train_index]\n    val = train.iloc[val_index]\n\n    # Training dataset\n    trn_ds = dataframe_to_dataset(trn, dummies=False)\n    trn_ds = trn_ds.batch(batch_size)\n    trn_ds = trn_ds.shuffle(buffer_size=300)\n    # Validation dataset\n    val_ds = dataframe_to_dataset(val, dummies=False)\n    val_ds = val_ds.batch(batch_size)\n\n    # Create the model\n    model = create_model(trn_ds, units, dropout, categorical_vocab, lr=lr, mode=\"regression\")\n    # Early stopping callback\n    early_stop_cb = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', patience=3, restore_best_weights=True\n    )\n    # Fit the model\n    model.fit(\n        trn_ds,\n        epochs=50,\n        validation_data=val_ds,\n        callbacks=[early_stop_cb]\n    )\n\n    # Get the out-of-fold predictions\n    fold_pred = model.predict(val_ds)\n    fold_pred = pd.DataFrame(fold_pred, columns=[\"scores\"])\n    fold_pred[\"idx\"] = val_index\n    # Combine with other folds\n    oof_pred = pd.concat([oof_pred, fold_pred])\n\n    # Store validation score\n    scores.append(early_stop_cb.best)\n\n    # Test predictions\n    pred = model.predict(test_ds)\n    preds.append(pred)\n\n# Order and set index\noof_pred = oof_pred.sort_values(by=[\"idx\"]).set_index(\"idx\")\n# Save the out-of-fold predictions\n#oof_pred.to_csv(\"./data/oof_pred_lstm.csv\", index=False)\nprint(f'The cross validation MSE is {np.mean(scores).round(4)}')\n\noof_lstm_reg = oof_pred\n\n# Combining test predictions\npreds = np.stack(preds, axis=1)\npreds = np.mean(preds, axis=1)\npreds = pd.DataFrame(preds, columns=[\"scores\"])\ntest_lstm_reg = preds","metadata":{"execution":{"iopub.status.busy":"2022-05-04T14:09:05.501832Z","iopub.execute_input":"2022-05-04T14:09:05.502401Z","iopub.status.idle":"2022-05-04T14:34:27.713049Z","shell.execute_reply.started":"2022-05-04T14:09:05.502364Z","shell.execute_reply":"2022-05-04T14:34:27.712081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classification","metadata":{}},{"cell_type":"code","source":"# Config\nbatch_size = 128\nlr = 0.00041445896727585396\nunits = [84, 43, 10]\ndropout = [0.45357535687931116, 0.19886952332377966, 0.06540638692985593]\n\n# Change target to home, away, draw\ntrain[\"target\"] = y_cat\n\n# Kfold cross validation\nkf = KFold(n_splits=10, shuffle=True, random_state=32)\nkf.get_n_splits(train)\noof_pred = pd.DataFrame()\nscores = []\n\n# Test\ntest_ds = dataframe_to_dataset(test, is_test=True)\ntest_ds = test_ds.batch(batch_size)\npreds = []\n\nfor train_index, val_index in kf.split(train):\n    trn = train.iloc[train_index]\n    val = train.iloc[val_index]\n\n    # Training dataset\n    trn_ds = dataframe_to_dataset(trn, dummies=True)\n    trn_ds = trn_ds.batch(batch_size)\n    trn_ds = trn_ds.shuffle(buffer_size=500)\n    # Validation dataset\n    val_ds = dataframe_to_dataset(val, dummies=True)\n    val_ds = val_ds.batch(batch_size)\n\n    # Create the model\n    model = create_model(trn_ds, units, dropout, categorical_vocab, lr=lr, mode=\"classification\")\n    # Early stopping callback\n    early_stop_cb = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', patience=3, restore_best_weights=True\n    )\n    # Fit the model\n    model.fit(\n        trn_ds,\n        epochs=50,\n        validation_data=val_ds,\n        callbacks=[early_stop_cb]\n    )\n\n    # Get the out-of-fold predictions\n    fold_pred = model.predict(val_ds)\n    fold_pred = pd.DataFrame(fold_pred, columns=[\"away\", \"draw\", \"home\"])\n    fold_pred[\"idx\"] = val_index\n    # Combine with other folds\n    oof_pred = pd.concat([oof_pred, fold_pred])\n\n    # Store validation score\n    scores.append(early_stop_cb.best)\n\n    # Test predictions\n    pred = model.predict(test_ds)\n    preds.append(pred)\n\n# Order and set index\noof_pred = oof_pred.sort_values(by=[\"idx\"]).set_index(\"idx\")\n# Save the out-of-fold predictions\n#oof_pred.to_csv(\"./data/oof_pred_lstm.csv\", index=False)\nprint(f'The cross validation log loss is {np.mean(scores).round(4)}')\n\noof_lstm = oof_pred\noof_lstm = oof_lstm.add_suffix(\"_lstm\")\n\n# Combining test predictions\npreds = np.stack(preds, axis=2)\npreds = np.mean(preds, axis=2)\npreds = pd.DataFrame(preds, columns=[\"away\", \"draw\", \"home\"])\ntest_lstm = preds\ntest_lstm = test_lstm.add_suffix(\"_lstm\")","metadata":{"papermill":{"duration":7234.906883,"end_time":"2022-04-19T04:29:13.892738","exception":false,"start_time":"2022-04-19T02:28:38.985855","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T14:40:09.838623Z","iopub.execute_input":"2022-05-04T14:40:09.838973Z","iopub.status.idle":"2022-05-04T14:49:38.615545Z","shell.execute_reply.started":"2022-05-04T14:40:09.838935Z","shell.execute_reply":"2022-05-04T14:49:38.614286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LGBM","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(train_path, low_memory=False)\ntest = pd.read_csv(test_path, low_memory=False)\n\ntrain = prepare_df(train, statistical_features=True)\ntest = prepare_df(test, statistical_features=True)\n\n# Add out-of-fold predicted scores\ntrain[\"scores\"] = oof_lstm_reg[\"scores\"]\ntest[\"scores\"] = test_lstm_reg[\"scores\"]\n\ncols_regex = \"target|mean|max|min|last|median|std|is_play_home|is_cup|league_id|same_league|streak|count|same_coach|^diff|scores\"\n\n\ntrain = train.filter(regex=cols_regex)\ntest = test.filter(regex=cols_regex)\n\ncat_features = list(train.filter(regex=\"league_id|result\"))\nle = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=9999)\nle = le.fit(train[cat_features])\ntrain[cat_features] = le.transform(train[cat_features])\ntest[cat_features] = le.transform(test[cat_features])\n\nle = LabelEncoder()\nlabels = train[\"target\"]\nle = le.fit(labels)\ntrain[\"target\"] = le.transform(train[\"target\"])\n\nkf = KFold(n_splits=10, shuffle=True, random_state=32)\nkf.get_n_splits(train)\noof_pred = pd.DataFrame()\nscores = []\n\n# Test\npreds = []\n\nfor train_index, val_index in kf.split(train):\n    x_trn = train.iloc[train_index]\n    y_trn = x_trn.pop(\"target\")\n    x_val = train.iloc[val_index]\n    y_val = x_val.pop(\"target\")\n\n    trn_data = lgb.Dataset(data = x_trn, label = y_trn)\n    val_data = lgb.Dataset(data = x_val, label = y_val)\n\n    param = {\n      'objective': \"multiclass\",\n      'num_classes': 3,\n      'boosting': \"gbdt\",\n      'learning_rate': 0.010642690092897651,\n      'max_depth': 6,\n      'subsample': 0.2,\n      'colsample_bytree': 0.2,\n      'num_threads': 4,\n      'num_leaves': 256,\n      'reg_alpha': 0,\n      'reg_lambda': 0,\n      'seed': 32,\n      'verbosity': -1\n    }\n\n    fit = lgb.train(\n      param,\n      trn_data,\n      #categorical_feature = cat_features,\n      num_boost_round=2000,\n      valid_sets=[trn_data, val_data],\n      callbacks = [lgb.early_stopping(20)]\n    )\n\n    fold_pred = fit.predict(x_val)\n    fold_pred = pd.DataFrame(fold_pred, columns=[\"away\", \"draw\", \"home\"])\n    fold_pred[\"idx\"] = val_index\n    # Combine with other folds\n    oof_pred = pd.concat([oof_pred, fold_pred])\n\n    # Store validation score\n    scores.append(fit.best_score['valid_1']['multi_logloss'])\n\n    # Test predictions\n    pred = fit.predict(test)\n    preds.append(pred)\n\n\n# Order and set index\noof_pred = oof_pred.sort_values(by=[\"idx\"]).set_index(\"idx\")\n# Save the out-of-fold predictions\nprint(f'The cross validation log loss is {np.mean(scores).round(4)}')\n\noof_lgbm = oof_pred\noof_lgbm = oof_lgbm.add_suffix(\"_lgbm\")\n\n# Combining test predictions\npreds = np.stack(preds, axis=2)\npreds = np.mean(preds, axis=2)\npreds = pd.DataFrame(preds, columns=[\"away\", \"draw\", \"home\"])\ntest_lgbm = preds\ntest_lgbm = test_lgbm.add_suffix(\"_lgbm\")","metadata":{"papermill":{"duration":2173.07012,"end_time":"2022-04-19T05:05:42.197948","exception":false,"start_time":"2022-04-19T04:29:29.127828","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-04T15:05:20.208905Z","iopub.execute_input":"2022-05-04T15:05:20.209195Z","iopub.status.idle":"2022-05-04T15:09:49.712187Z","shell.execute_reply.started":"2022-05-04T15:05:20.209163Z","shell.execute_reply":"2022-05-04T15:09:49.710041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Blend","metadata":{}},{"cell_type":"code","source":"y = pd.read_csv(train_path, usecols=[\"target\"])\n\nbest_score = 10\nbest_p = 1\n\nfor p in np.arange(0, 1.01, 0.01):\n    score = metrics.log_loss(y[88750:], p * oof_lstm[88750:].values + (1 - p) * oof_lgbm[88750:].values)\n    if score < best_score:\n        best_score = score\n        best_p = p    \n\nprint(f'The best p is {best_p}')\nprint(f'The best score is {best_score}')\np = best_p\npred = p * test_lstm.values + (1 - p) * test_lgbm.values\npred = pd.DataFrame(pred, columns=[\"away\", \"draw\", \"home\"])\n\nsubmission = pd.read_csv(test_path, usecols=[\"id\"])\nsubmission = pd.concat([submission, pred], axis=1)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.execute_input":"2022-04-19T05:06:13.194638Z","iopub.status.busy":"2022-04-19T05:06:13.194303Z","iopub.status.idle":"2022-04-19T05:06:45.646794Z","shell.execute_reply":"2022-04-19T05:06:45.64583Z"},"papermill":{"duration":47.893548,"end_time":"2022-04-19T05:06:45.649064","exception":false,"start_time":"2022-04-19T05:05:57.755516","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}