{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nimport datetime as dt\nimport os\nimport random\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pylab as plt\nfrom sklearn import metrics\nfrom sklearn import impute\nfrom sklearn import preprocessing\nfrom sklearn.utils import class_weight\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nplt.style.use('ggplot')\nplt.rcParams['figure.figsize'] = (10, 8)\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', 7)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-15T12:06:57.273024Z","iopub.execute_input":"2022-02-15T12:06:57.273337Z","iopub.status.idle":"2022-02-15T12:06:57.281091Z","shell.execute_reply.started":"2022-02-15T12:06:57.273303Z","shell.execute_reply":"2022-02-15T12:06:57.280125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = Path('../input/football-match-probability-prediction')\n!ls {PATH}","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:06:57.283219Z","iopub.execute_input":"2022-02-15T12:06:57.283494Z","iopub.status.idle":"2022-02-15T12:06:58.001415Z","shell.execute_reply.started":"2022-02-15T12:06:57.283460Z","shell.execute_reply":"2022-02-15T12:06:58.000446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain = pd.read_csv(PATH / 'train.csv')\ntest = pd.read_csv(PATH / 'test.csv')\n\nfor col in train.filter(regex='date', axis=1).columns:\n    train[col] = pd.to_datetime(train[col])\n    test[col] = pd.to_datetime(test[col])\n    \n# date based features\nfor i in range(1, 11):\n    train[f'home_team_history_match_days_ago_{i}'] = (train['match_date'] - train[f'home_team_history_match_date_{i}']).dt.days\n    train[f'away_team_history_match_days_ago_{i}'] = (train['match_date'] - train[f'away_team_history_match_date_{i}']).dt.days\n    test[f'home_team_history_match_days_ago_{i}'] = (test['match_date'] - test[f'home_team_history_match_date_{i}']).dt.days\n    test[f'away_team_history_match_days_ago_{i}'] = (test['match_date'] - test[f'away_team_history_match_date_{i}']).dt.days\n    \n# remove two matchs with possible error\ntrain = train[train['home_team_name'] != train['away_team_name']].reset_index(drop=True)\n    \ntrain","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:06:58.003615Z","iopub.execute_input":"2022-02-15T12:06:58.004123Z","iopub.status.idle":"2022-02-15T12:07:07.301949Z","shell.execute_reply.started":"2022-02-15T12:06:58.004078Z","shell.execute_reply":"2022-02-15T12:07:07.301299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:07.303355Z","iopub.execute_input":"2022-02-15T12:07:07.303789Z","iopub.status.idle":"2022-02-15T12:07:07.324671Z","shell.execute_reply.started":"2022-02-15T12:07:07.303753Z","shell.execute_reply":"2022-02-15T12:07:07.323943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:07.326835Z","iopub.execute_input":"2022-02-15T12:07:07.327094Z","iopub.status.idle":"2022-02-15T12:07:07.345290Z","shell.execute_reply.started":"2022-02-15T12:07:07.327052Z","shell.execute_reply":"2022-02-15T12:07:07.344533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Choosing a validation method","metadata":{}},{"cell_type":"code","source":"print('Train:')\nprint('First match at:', train['match_date'].min())\nprint('Last match at: ', train['match_date'].max())\nprint('Range:', train['match_date'].max() - train['match_date'].min())","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:07.346387Z","iopub.execute_input":"2022-02-15T12:07:07.346891Z","iopub.status.idle":"2022-02-15T12:07:07.357133Z","shell.execute_reply.started":"2022-02-15T12:07:07.346850Z","shell.execute_reply":"2022-02-15T12:07:07.356260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Test:')\nprint('First match at:', test['match_date'].min())\nprint('Last match at: ', test['match_date'].max())\nprint('Range:', test['match_date'].max() - test['match_date'].min())","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:07.358677Z","iopub.execute_input":"2022-02-15T12:07:07.359018Z","iopub.status.idle":"2022-02-15T12:07:07.369367Z","shell.execute_reply.started":"2022-02-15T12:07:07.358981Z","shell.execute_reply":"2022-02-15T12:07:07.368642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try to take 213 days to use on validation set","metadata":{}},{"cell_type":"code","source":"validation_split = dt.datetime.strptime(\n    '2021-05-01 00:00:00', '%Y-%m-%d %H:%M:%S') - dt.timedelta(days=213, hours=23, minutes=15)\nprint('Train until:', validation_split)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:07.370381Z","iopub.execute_input":"2022-02-15T12:07:07.370980Z","iopub.status.idle":"2022-02-15T12:07:07.377043Z","shell.execute_reply.started":"2022-02-15T12:07:07.370943Z","shell.execute_reply":"2022-02-15T12:07:07.376216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training a LSTM model","metadata":{}},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    classes = 3\n    use_class_weights = False\n    batch_size = 512\n    epochs = 100\n    early_stopping_patience = epochs // 10\n    mask = -999.0\n    timesteps = 10\n    target = 'target_int'\n    historical_features = [\n        # home based\n        'home_team_history_goal',\n        'home_team_history_opponent_goal',\n        'home_team_history_is_play_home', \n        'home_team_history_rating',\n        'home_team_history_opponent_rating',\n        'home_team_history_match_days_ago',\n        \n        # away based\n        'away_team_history_goal', \n        'away_team_history_opponent_goal',\n        'away_team_history_rating',\n        'away_team_history_opponent_rating',\n        'away_team_history_match_days_ago'\n    ] \n    \ndef set_seed(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\ndef wide_to_long(df, feature: str, timesteps: int, mask=CFG.mask):\n    df_ = df.copy()\n    if feature not in CFG.historical_features:\n        features = [feature]\n    else:\n        features = [f'{feature}_{i}' for i in range(1, timesteps + 1)][::-1]\n    df_ = df_[['id'] + features]\n    df_ = df_.fillna(mask)\n    series = df_.set_index('id').stack().reset_index(level=1)[0].rename(feature)\n    return series","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:07.378370Z","iopub.execute_input":"2022-02-15T12:07:07.379144Z","iopub.status.idle":"2022-02-15T12:07:07.390774Z","shell.execute_reply.started":"2022-02-15T12:07:07.379108Z","shell.execute_reply":"2022-02-15T12:07:07.389917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# maps\ntarget2int = {'away': 0, 'draw': 1, 'home': 2}\nint2target = {x[1]: x[0] for x in target2int.items()}\n\n# encode target\ntrain['target_int'] = train['target'].map(target2int)\n\nX_test = test.copy()\n\n# split train/val\nX_train = train[train['match_date'] <= validation_split].reset_index(drop=True)\nX_val = train[train['match_date'] > validation_split].reset_index(drop=True)\n\n# preprocess\nfeatures_pattern = '_[0-9]|'.join(CFG.historical_features) + '_[0-9]'\nfeatures_to_preprocess = train.filter(regex=features_pattern, axis=1).columns.tolist()\nprint('Features to preprocess:', len(features_to_preprocess))\n# this Scaler removes the median and scales the data according to the quantile range (defaults to IQR: Interquartile Range)\nscaler = preprocessing.RobustScaler()\nX_train_pre = pd.DataFrame(scaler.fit_transform(X_train[features_to_preprocess]), columns=features_to_preprocess)\nX_train = pd.concat([X_train[['id', 'league_name', 'target_int']], X_train_pre], axis=1)\nX_val_pre = pd.DataFrame(scaler.transform(X_val[features_to_preprocess]), columns=features_to_preprocess)\nX_val = pd.concat([X_val[['id', 'league_name', 'target_int']], X_val_pre], axis=1)\nX_test_pre = pd.DataFrame(scaler.transform(X_test[features_to_preprocess]), columns=features_to_preprocess)\nX_test = pd.concat([X_test[['id', 'league_name']], X_test_pre], axis=1)\n\n# create targets\ny_train = wide_to_long(X_train, 'target_int', timesteps=CFG.timesteps).values.reshape(-1, 1)\ny_val = wide_to_long(X_val, 'target_int', timesteps=CFG.timesteps).values.reshape(-1, 1)\n\n# create historical features\nX_train = pd.concat([\n    wide_to_long(X_train, feature=feature, timesteps=CFG.timesteps) for feature in CFG.historical_features\n], axis=1).values.reshape(-1, CFG.timesteps, len(CFG.historical_features))\nX_val = pd.concat([\n    wide_to_long(X_val, feature=feature, timesteps=CFG.timesteps) for feature in CFG.historical_features\n], axis=1).values.reshape(-1, CFG.timesteps, len(CFG.historical_features))\nX_test = pd.concat([\n    wide_to_long(X_test, feature=feature, timesteps=CFG.timesteps) for feature in CFG.historical_features\n], axis=1).values.reshape(-1, CFG.timesteps, len(CFG.historical_features))\n\nprint(X_train.shape)\nprint(y_train.shape)\nprint()\nprint(X_val.shape)\nprint(y_val.shape)\nprint()\nprint(X_test.shape)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-15T12:07:07.391948Z","iopub.execute_input":"2022-02-15T12:07:07.392552Z","iopub.status.idle":"2022-02-15T12:07:09.809225Z","shell.execute_reply.started":"2022-02-15T12:07:07.392511Z","shell.execute_reply":"2022-02-15T12:07:09.808512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distribution of away, draw, home\nprint(np.bincount(y_train.ravel()) / len(y_train))\nprint(np.bincount(y_val.ravel()) / len(y_val))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:09.812186Z","iopub.execute_input":"2022-02-15T12:07:09.812411Z","iopub.status.idle":"2022-02-15T12:07:09.820024Z","shell.execute_reply.started":"2022-02-15T12:07:09.812383Z","shell.execute_reply":"2022-02-15T12:07:09.819162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `draw` class is less frequent than the others.  ","metadata":{}},{"cell_type":"code","source":"if CFG.use_class_weights:\n    # https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\n    class_weights = class_weight.compute_class_weight(\n        class_weight='balanced',\n        classes=np.unique(y_train),\n        y=y_train.ravel()\n    )\n    class_weights = {i: x for i, x in enumerate(class_weights)}\nelse:\n    class_weights = None\nprint('Class weights:', class_weights)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:09.821396Z","iopub.execute_input":"2022-02-15T12:07:09.821775Z","iopub.status.idle":"2022-02-15T12:07:09.829118Z","shell.execute_reply.started":"2022-02-15T12:07:09.821739Z","shell.execute_reply":"2022-02-15T12:07:09.828216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n\n    # input\n    input_ = tf.keras.Input(shape=X_train.shape[1:])\n    mask = tf.keras.layers.Masking(mask_value=CFG.mask, input_shape=(X_train.shape[1:]))(input_)\n    x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(16, return_sequences=True, activation='tanh'))(mask)\n    x = tf.keras.layers.Dropout(0.5)(x)  \n    x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(8, return_sequences=True, activation='tanh'))(x)\n    x = tf.keras.layers.Dropout(0.5)(x)\n    x = tf.keras.layers.Flatten()(x)\n\n    # output\n    output = tf.keras.layers.Dense(CFG.classes, activation='softmax')(x)\n    \n    model = tf.keras.Model(\n        inputs=[input_],\n        outputs=[output]\n    )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:09.830703Z","iopub.execute_input":"2022-02-15T12:07:09.831410Z","iopub.status.idle":"2022-02-15T12:07:09.840781Z","shell.execute_reply.started":"2022-02-15T12:07:09.831372Z","shell.execute_reply":"2022-02-15T12:07:09.839862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nset_seed(CFG.seed)\n\n# callbacks\nes = tf.keras.callbacks.EarlyStopping(\n    patience=CFG.early_stopping_patience,\n    restore_best_weights=True,\n    verbose=1\n)\nrlrop = tf.keras.callbacks.ReduceLROnPlateau(\n    factor=0.8,\n    patience=CFG.early_stopping_patience // 2,\n    verbose=1\n)\n\nmodel = create_model()\nmodel.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics='accuracy')\n\n# fit\nh = model.fit(\n    X_train,\n    y_train,\n    validation_data=(X_val, y_val),\n    batch_size=CFG.batch_size,\n    epochs=CFG.epochs,\n    callbacks=[\n        es,\n        rlrop\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T12:07:09.842262Z","iopub.execute_input":"2022-02-15T12:07:09.842819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(h.history['loss'], label='Train')\nplt.plot(h.history['val_loss'], label='Val')\nplt.legend();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate","metadata":{}},{"cell_type":"code","source":"def evaluate(model, X, y):\n    probs = model.predict(X)\n    preds = np.argmax(probs, axis=1)\n    report = metrics.classification_report(y, preds)\n    print(report)\n    logloss = metrics.log_loss(y, probs)\n    print('Log loss:', logloss)\n    cm = metrics.confusion_matrix(y, preds)\n    disp = metrics.ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=list(target2int.keys()))\n    disp.plot(cmap=plt.cm.Blues)\n    plt.show();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate(model, X_val, y_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems that the `draw` class is hurting the log loss.   ","metadata":{}},{"cell_type":"markdown","source":"## Predict on test set","metadata":{}},{"cell_type":"code","source":"# away, draw, home\nprobs_test = model.predict(X_test)\nprobs_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.DataFrame({\n    'id': test['id'],\n    'away': probs_test[:, 0],\n    'draw': probs_test[:, 1],\n    'home': probs_test[:, 2]\n})\ndf_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.concat([\n    train['target_int'].value_counts(normalize=True).sort_index().rename('observed'),\n    pd.Series(np.argmax(df_sub.iloc[:, 1:].values, axis=1), name='predicted').value_counts(normalize=True).sort_index()\n], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Very few predicted draws compared to the observed target.   ","metadata":{}},{"cell_type":"code","source":"df_sub.iloc[:, 1:].plot.hist(alpha=0.5);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission expects this order: id, home, away, draw\ndf_sub[['id', 'home', 'away', 'draw']].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}