{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nimport datetime as dt\nimport os\nimport random\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pylab as plt\nfrom sklearn import metrics\nfrom sklearn import impute\nfrom sklearn import preprocessing\nfrom sklearn.utils import class_weight\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nplt.style.use('ggplot')\nplt.rcParams['figure.figsize'] = (10, 8)\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', 7)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-12T23:57:31.527878Z","iopub.execute_input":"2022-03-12T23:57:31.528471Z","iopub.status.idle":"2022-03-12T23:57:37.66127Z","shell.execute_reply.started":"2022-03-12T23:57:31.528388Z","shell.execute_reply":"2022-03-12T23:57:37.660469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = Path('../input/football-match-probability-prediction')\n!ls {PATH}","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-03-12T23:57:37.66376Z","iopub.execute_input":"2022-03-12T23:57:37.664951Z","iopub.status.idle":"2022-03-12T23:57:38.33747Z","shell.execute_reply.started":"2022-03-12T23:57:37.664904Z","shell.execute_reply":"2022-03-12T23:57:38.336615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code forked from https://www.kaggle.com/igorkf\n\nhttps://www.kaggle.com/igorkf/football-match-probability-prediction-lstm-starter","metadata":{}},{"cell_type":"code","source":"%%time\n\ntrain = pd.read_csv(PATH / 'train.csv')\ntest = pd.read_csv(PATH / 'test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-12T23:57:38.340625Z","iopub.execute_input":"2022-03-12T23:57:38.341306Z","iopub.status.idle":"2022-03-12T23:57:47.42809Z","shell.execute_reply.started":"2022-03-12T23:57:38.341273Z","shell.execute_reply":"2022-03-12T23:57:47.427318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train.filter(regex='date', axis=1).columns:\n    train[col] = pd.to_datetime(train[col])\n    test[col] = pd.to_datetime(test[col])\n    \n# date based features\nfor i in range(1, 11):\n    train[f'home_team_history_match_days_ago_{i}'] = (train['match_date'] - train[f'home_team_history_match_date_{i}']).dt.days\n    train[f'away_team_history_match_days_ago_{i}'] = (train['match_date'] - train[f'away_team_history_match_date_{i}']).dt.days\n    test[f'home_team_history_match_days_ago_{i}'] = (test['match_date'] - test[f'home_team_history_match_date_{i}']).dt.days\n    test[f'away_team_history_match_days_ago_{i}'] = (test['match_date'] - test[f'away_team_history_match_date_{i}']).dt.days\n    \n# remove two matchs with possible error\ncombined_train=train\ncombined_train['tag']='Train_Data'\n\ncombined_test=test\ncombined_test['tag']='Test_Data'\n\ncombined=combined_train.append(combined_test)\ncombined['year']=pd.to_datetime(combined['match_date']).dt.year\n\nimport datetime as dt\nrating_features=[]\ncombined['date_difference_last_match_home']=(pd.to_datetime(combined['match_date'])-pd.to_datetime(combined['home_team_history_match_date_1'])).dt.days\ncombined['date_difference_last_match_away']=(pd.to_datetime(combined['match_date'])-pd.to_datetime(combined['away_team_history_match_date_1'])).dt.days\n\ncombined['Goals_For_Home']=combined['home_team_history_goal_1']+combined['home_team_history_goal_2']+combined['home_team_history_goal_3']+combined['home_team_history_goal_4']+combined['home_team_history_goal_5']\ncombined['Opponent_Goals_For_Home']=combined['home_team_history_opponent_goal_1']+combined['home_team_history_opponent_goal_2']+combined['home_team_history_opponent_goal_3']+combined['home_team_history_opponent_goal_4']+combined['home_team_history_opponent_goal_5']\ncombined['goal_difference']=combined['Goals_For_Home']-combined['Opponent_Goals_For_Home']\n\ncombined['Goals_For_Away']=combined['away_team_history_goal_1']+combined['away_team_history_goal_2']+combined['away_team_history_goal_3']+combined['away_team_history_goal_4']+combined['away_team_history_goal_5']\ncombined['Opponent_Goals_For_Away']=combined['away_team_history_opponent_goal_1']+combined['away_team_history_opponent_goal_2']+combined['away_team_history_opponent_goal_3']+combined['away_team_history_opponent_goal_4']+combined['away_team_history_opponent_goal_5']\ncombined['goal_difference_away']=combined['Goals_For_Away']-combined['Opponent_Goals_For_Away']\n\n#Performance in Last 5 matches\nfor i in range(1,11):\n    combined['last_match_result_home_'+str(i)]=np.where(combined['home_team_history_goal_'+str(i)]>combined['home_team_history_opponent_goal_'+str(i)],3,np.where(combined['home_team_history_goal_'+str(i)]<combined['home_team_history_opponent_goal_'+str(i)],0,1))\n    rating_features.append('last_match_result_home_'+str(i))\n    \n    combined['last_match_result_away_'+str(i)]=np.where(combined['away_team_history_goal_'+str(i)]>combined['away_team_history_opponent_goal_'+str(i)],0,np.where(combined['away_team_history_goal_'+str(i)]<combined['away_team_history_opponent_goal_'+str(i)],3,1))\n    rating_features.append('last_match_result_away_'+str(i))\n\ncombined['home_performance_last_5']=combined['last_match_result_home_1']+combined['last_match_result_home_2']+combined['last_match_result_home_3']+combined['last_match_result_home_4']+combined['last_match_result_home_5']\ncombined['away_performance_last_5']=combined['last_match_result_away_1']+combined['last_match_result_away_2']+combined['last_match_result_away_3']+combined['last_match_result_away_4']+combined['last_match_result_away_5']\ncombined['home_vs_away']=combined['home_performance_last_5']+combined['away_performance_last_5']\n\n\nrating_features=rating_features+['home_performance_last_5','away_performance_last_5','home_vs_away']\n\ndef streak(m1,m2,m3,m4,m5,k):\n    streak=0\n    \n    lst_results=[m1,m2,m3,m4,m5]\n    for i in lst_results:\n        if(i==k):\n            streak=streak+1\n        else:\n            return streak\n            break\n    \n    return streak\n\n\ncombined['home_same_coach']=combined.apply(lambda x:streak(x.home_team_history_coach_1,x.home_team_history_coach_2,x.home_team_history_coach_3,x.home_team_history_coach_4,x.home_team_history_coach_5,x.home_team_coach_id),axis=1)\ncombined['away_same_coach']=combined.apply(lambda x:streak(x.away_team_history_coach_1,x.away_team_history_coach_2,x.away_team_history_coach_3,x.away_team_history_coach_4,x.away_team_history_coach_5,x.away_team_coach_id),axis=1)\n\n\n\nmapping_freq=40\n\nfrequencies=combined['home_team_name'].value_counts()\nmapping=combined['home_team_name'].map(frequencies)\ncombined['home_team_name_mapping']=combined['home_team_name'].mask(mapping<mapping_freq,'Other')\n\nfrequencies=combined['away_team_name'].value_counts()\nmapping=combined['away_team_name'].map(frequencies)\ncombined['away_team_name_mapping']=combined['away_team_name'].mask(mapping<mapping_freq,'Other')\n\ncombined['away_coach_change']=np.where(combined['away_team_coach_id']==combined['away_team_history_coach_1'],0,1)\n\n\ncombined['home_team_coach_id']=combined['home_team_coach_id'].astype('str')\nfrequencies=combined['home_team_name'].value_counts()\nmapping=combined['home_team_name'].map(frequencies)\ncombined['home_team_name_mapping']=combined['home_team_name'].mask(mapping<mapping_freq,'Other')\n\n\n# Coach Mapping\ncombined['home_team_coach_id']=combined['home_team_coach_id'].astype('str')\nfrequencies=combined['home_team_coach_id'].value_counts()\nmapping=combined['home_team_coach_id'].map(frequencies)\ncombined['home_team_coach_mapping']=combined['home_team_coach_id'].mask(mapping<mapping_freq,'Other')\n\ncombined['away_team_coach_id']=combined['away_team_coach_id'].astype('str')\nfrequencies=combined['away_team_coach_id'].value_counts()\nmapping=combined['away_team_coach_id'].map(frequencies)\ncombined['away_team_coach_mapping']=combined['away_team_coach_id'].mask(mapping<mapping_freq,'Other')\n\n\n\n\ncombined['home_winning_streak']=combined.apply(lambda x:streak(x.last_match_result_home_1,x.last_match_result_home_2,x.last_match_result_home_3,x.last_match_result_home_4,x.last_match_result_home_5,3),axis=1)\ncombined['home_losing_streak']=combined.apply(lambda x:streak(x.last_match_result_home_1,x.last_match_result_home_2,x.last_match_result_home_3,x.last_match_result_home_4,x.last_match_result_home_5,0),axis=1)\ncombined['home_draw_streak']=combined.apply(lambda x:streak(x.last_match_result_home_1,x.last_match_result_home_2,x.last_match_result_home_3,x.last_match_result_home_4,x.last_match_result_home_5,1),axis=1)\n\ncombined['away_winning_streak']=combined.apply(lambda x:streak(x.last_match_result_away_1,x.last_match_result_away_2,x.last_match_result_away_3,x.last_match_result_away_4,x.last_match_result_away_5,3),axis=1)\ncombined['away_losing_streak']=combined.apply(lambda x:streak(x.last_match_result_away_1,x.last_match_result_away_2,x.last_match_result_away_3,x.last_match_result_away_4,x.last_match_result_away_5,0),axis=1)\ncombined['away_draw_streak']=combined.apply(lambda x:streak(x.last_match_result_away_1,x.last_match_result_away_2,x.last_match_result_away_3,x.last_match_result_away_4,x.last_match_result_away_5,1),axis=1)\n# combined=pd.get_dummies(combined)\ntrain=combined[combined['tag']=='Train_Data']\ntest=combined[combined['tag']=='Test_Data']\ncombined","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-03-12T23:58:09.162249Z","iopub.execute_input":"2022-03-12T23:58:09.162661Z","iopub.status.idle":"2022-03-13T00:02:25.434475Z","shell.execute_reply.started":"2022-03-12T23:58:09.162626Z","shell.execute_reply":"2022-03-13T00:02:25.43382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rating_features=['date_difference_last_match_home',\n       'date_difference_last_match_away', 'Goals_For_Home',\n       'Opponent_Goals_For_Home', 'goal_difference', 'Goals_For_Away',\n       'Opponent_Goals_For_Away', 'goal_difference_away',\n       'last_match_result_home_1', 'last_match_result_away_1',\n       'last_match_result_home_2', 'last_match_result_away_2',\n       'last_match_result_home_3', 'last_match_result_away_3',\n       'last_match_result_home_4', 'last_match_result_away_4',\n       'last_match_result_home_5', 'last_match_result_away_5',\n       'home_performance_last_5', 'away_performance_last_5', 'home_vs_away',\n       'home_same_coach', 'away_same_coach', 'away_coach_change',\n       \n       'home_winning_streak', 'home_losing_streak', 'home_draw_streak',\n       'away_winning_streak', 'away_losing_streak', 'away_draw_streak']","metadata":{"execution":{"iopub.status.busy":"2022-03-13T00:02:25.435855Z","iopub.execute_input":"2022-03-13T00:02:25.436079Z","iopub.status.idle":"2022-03-13T00:02:25.44296Z","shell.execute_reply.started":"2022-03-13T00:02:25.436048Z","shell.execute_reply":"2022-03-13T00:02:25.442047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined","metadata":{"execution":{"iopub.status.busy":"2022-03-13T00:02:25.444618Z","iopub.execute_input":"2022-03-13T00:02:25.445275Z","iopub.status.idle":"2022-03-13T00:02:25.67609Z","shell.execute_reply.started":"2022-03-13T00:02:25.445232Z","shell.execute_reply":"2022-03-13T00:02:25.675416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Choosing a validation method","metadata":{}},{"cell_type":"code","source":"print('Train:')\nprint('First match at:', train['match_date'].min())\nprint('Last match at: ', train['match_date'].max())\nprint('Range:', train['match_date'].max() - train['match_date'].min())","metadata":{"execution":{"iopub.status.busy":"2022-03-13T00:02:25.677657Z","iopub.execute_input":"2022-03-13T00:02:25.678849Z","iopub.status.idle":"2022-03-13T00:02:25.690466Z","shell.execute_reply.started":"2022-03-13T00:02:25.67881Z","shell.execute_reply":"2022-03-13T00:02:25.689498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Test:')\nprint('First match at:', test['match_date'].min())\nprint('Last match at: ', test['match_date'].max())\nprint('Range:', test['match_date'].max() - test['match_date'].min())","metadata":{"execution":{"iopub.status.busy":"2022-03-13T00:02:25.692214Z","iopub.execute_input":"2022-03-13T00:02:25.692465Z","iopub.status.idle":"2022-03-13T00:02:25.702005Z","shell.execute_reply.started":"2022-03-13T00:02:25.692432Z","shell.execute_reply":"2022-03-13T00:02:25.700992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try to take 213 days to use on validation set","metadata":{}},{"cell_type":"code","source":"validation_split = dt.datetime.strptime(\n    '2021-05-01 00:00:00', '%Y-%m-%d %H:%M:%S') - dt.timedelta(days=50, hours=23, minutes=15)\nprint('Train until:', validation_split)","metadata":{"execution":{"iopub.status.busy":"2022-03-13T00:02:25.703775Z","iopub.execute_input":"2022-03-13T00:02:25.704398Z","iopub.status.idle":"2022-03-13T00:02:25.712089Z","shell.execute_reply.started":"2022-03-13T00:02:25.70436Z","shell.execute_reply":"2022-03-13T00:02:25.71137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training a LSTM model","metadata":{}},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    classes = 3\n    use_class_weights = False\n    batch_size = 512\n    epochs = 100\n    early_stopping_patience = epochs // 10\n    mask = -999.0\n    timesteps = 10\n    target = 'target_int'\n    historical_features = [\n        # home based\n        'home_team_history_goal',\n        'home_team_history_opponent_goal',\n        'home_team_history_is_play_home', \n        'home_team_history_rating',\n        'home_team_history_opponent_rating',\n        'home_team_history_match_days_ago',\n        \n        # away based\n        'away_team_history_goal', \n        'away_team_history_opponent_goal',\n        'away_team_history_rating',\n        'away_team_history_opponent_rating',\n        'away_team_history_match_days_ago',\n\n    ] \n    \ndef set_seed(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\ndef wide_to_long(df, feature: str, timesteps: int, mask=CFG.mask):\n    df_ = df.copy()\n    if feature not in CFG.historical_features:\n        features = [feature]\n    else:\n        features = [f'{feature}_{i}' for i in range(1, timesteps + 1)][::-1]\n#     print(features)\n    df_ = df_[['id'] + features]\n    df_ = df_.fillna(mask)\n    series = df_.set_index('id').stack().reset_index(level=1)[0].rename(feature)\n    return series","metadata":{"execution":{"iopub.status.busy":"2022-03-06T18:38:12.63942Z","iopub.execute_input":"2022-03-06T18:38:12.639684Z","iopub.status.idle":"2022-03-06T18:38:12.650004Z","shell.execute_reply.started":"2022-03-06T18:38:12.639654Z","shell.execute_reply":"2022-03-06T18:38:12.648729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# maps\ntarget2int = {'away': 0, 'draw': 1, 'home': 2}\nint2target = {x[1]: x[0] for x in target2int.items()}\n\n# encode target\ntrain['target_int'] = train['target'].map(target2int)\n\nX_test = test.copy()\n\n# split train/val\nX_train = train[train['match_date'] <= validation_split].reset_index(drop=True)\nX_val = train[train['match_date'] > validation_split].reset_index(drop=True)\n\n# preprocess\nfeatures_pattern = '_[0-9]|'.join(CFG.historical_features) + '_[0-9]'\nfeatures_to_preprocess = train.filter(regex=features_pattern, axis=1).columns.tolist()\n\nfeatures_to_preprocess=features_to_preprocess+rating_features\n    \nprint('Features to preprocess:', len(features_to_preprocess))\n# this Scaler removes the median and scales the data according to the quantile range (defaults to IQR: Interquartile Range)\nscaler = preprocessing.RobustScaler()\nX_train_pre = pd.DataFrame(scaler.fit_transform(X_train[features_to_preprocess]), columns=features_to_preprocess)\nX_train_pre=pd.concat([X_train[['home_team_coach_mapping','away_team_coach_mapping','home_team_name_mapping','away_team_name_mapping']], X_train_pre], axis=1)\nX_train_pre=pd.get_dummies(X_train_pre)\n\nX_train = pd.concat([X_train[['id', 'league_name', 'target_int','home_team_coach_mapping']], X_train_pre], axis=1)\n\nX_val_pre = pd.DataFrame(scaler.transform(X_val[features_to_preprocess]), columns=features_to_preprocess)\nX_val_pre=pd.concat([X_val[['home_team_coach_mapping','away_team_coach_mapping','home_team_name_mapping','away_team_name_mapping']], X_val_pre], axis=1)\nX_val_pre=pd.get_dummies(X_val_pre)\n\nX_val = pd.concat([X_val[['id', 'league_name', 'target_int']], X_val_pre], axis=1)\nX_train,X_val = X_train.align(X_val, join='outer', axis=1, fill_value=0)\n\nX_test_pre = pd.DataFrame(scaler.transform(X_test[features_to_preprocess]), columns=features_to_preprocess)\nX_test_pre=pd.concat([X_test[['home_team_coach_mapping','away_team_coach_mapping','home_team_name_mapping','away_team_name_mapping']], X_test_pre], axis=1)\nX_test_pre=pd.get_dummies(X_test_pre)\n\nX_test = pd.concat([X_test[['id', 'league_name']], X_test_pre], axis=1)\nX_train,X_test = X_train.align(X_test, join='outer', axis=1, fill_value=0)\n\n\n# create targets\ny_train = wide_to_long(X_train, 'target_int', timesteps=CFG.timesteps).values.reshape(-1, 1)\ny_val = wide_to_long(X_val, 'target_int', timesteps=CFG.timesteps).values.reshape(-1, 1)\n\n# create historical features\nX_train = pd.concat([\n    wide_to_long(X_train, feature=feature, timesteps=CFG.timesteps) for feature in CFG.historical_features\n], axis=1).values.reshape(-1, CFG.timesteps, len(CFG.historical_features))\nX_val = pd.concat([\n    wide_to_long(X_val, feature=feature, timesteps=CFG.timesteps) for feature in CFG.historical_features\n], axis=1).values.reshape(-1, CFG.timesteps, len(CFG.historical_features))\nX_test = pd.concat([\n    wide_to_long(X_test, feature=feature, timesteps=CFG.timesteps) for feature in CFG.historical_features\n], axis=1).values.reshape(-1, CFG.timesteps, len(CFG.historical_features))\n\nprint(X_train.shape)\nprint(y_train.shape)\nprint()\nprint(X_val.shape)\nprint(y_val.shape)\nprint()\nprint(X_test.shape)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-06T18:38:14.684658Z","iopub.execute_input":"2022-03-06T18:38:14.68524Z","iopub.status.idle":"2022-03-06T18:38:23.686192Z","shell.execute_reply.started":"2022-03-06T18:38:14.685201Z","shell.execute_reply":"2022-03-06T18:38:23.684292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distribution of away, draw, home\nprint(np.bincount(y_train.ravel()) / len(y_train))\nprint(np.bincount(y_val.ravel()) / len(y_val))","metadata":{"execution":{"iopub.status.busy":"2022-03-06T18:38:23.687889Z","iopub.execute_input":"2022-03-06T18:38:23.688811Z","iopub.status.idle":"2022-03-06T18:38:23.696824Z","shell.execute_reply.started":"2022-03-06T18:38:23.68877Z","shell.execute_reply":"2022-03-06T18:38:23.695889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `draw` class is less frequent than the others.  ","metadata":{}},{"cell_type":"code","source":"if CFG.use_class_weights:\n    # https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\n    class_weights = class_weight.compute_class_weight(\n        class_weight='balanced',\n        classes=np.unique(y_train),\n        y=y_train.ravel()\n    )\n    class_weights = {i: x for i, x in enumerate(class_weights)}\nelse:\n    class_weights = None\nprint('Class weights:', class_weights)","metadata":{"execution":{"iopub.status.busy":"2022-03-06T18:38:23.698076Z","iopub.execute_input":"2022-03-06T18:38:23.698333Z","iopub.status.idle":"2022-03-06T18:38:23.707546Z","shell.execute_reply.started":"2022-03-06T18:38:23.698298Z","shell.execute_reply":"2022-03-06T18:38:23.706716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n\n    # input\n    input_ = tf.keras.Input(shape=X_train.shape[1:])\n    mask = tf.keras.layers.Masking(mask_value=CFG.mask, input_shape=(X_train.shape[1:]))(input_)\n    x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True, activation='tanh'))(mask)\n    x = tf.keras.layers.Dropout(0.5)(x)  \n    x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(16, return_sequences=True, activation='tanh'))(x)\n    x = tf.keras.layers.Dropout(0.5)(x)\n    x = tf.keras.layers.Flatten()(x)\n\n    # output\n    output = tf.keras.layers.Dense(CFG.classes, activation='softmax')(x)\n    \n    model = tf.keras.Model(\n        inputs=[input_],\n        outputs=[output]\n    )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-03-06T18:46:12.636517Z","iopub.execute_input":"2022-03-06T18:46:12.636791Z","iopub.status.idle":"2022-03-06T18:46:12.645529Z","shell.execute_reply.started":"2022-03-06T18:46:12.636762Z","shell.execute_reply":"2022-03-06T18:46:12.644403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nset_seed(CFG.seed)\n\n# callbacks\nes = tf.keras.callbacks.EarlyStopping(\n    patience=CFG.early_stopping_patience,\n    restore_best_weights=True,\n    verbose=1\n)\nrlrop = tf.keras.callbacks.ReduceLROnPlateau(\n    factor=0.8,\n    patience=CFG.early_stopping_patience // 2,\n    verbose=1\n)\n\nmodel = create_model()\nmodel.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics='accuracy')\n\n# fit\nh = model.fit(\n    X_train,\n    y_train,\n    validation_data=(X_val, y_val),\n    batch_size=CFG.batch_size,\n    epochs=CFG.epochs,\n    callbacks=[\n        es,\n        rlrop\n    ]\n)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-03-06T18:41:13.200877Z","iopub.execute_input":"2022-03-06T18:41:13.201394Z","iopub.status.idle":"2022-03-06T18:43:00.554235Z","shell.execute_reply.started":"2022-03-06T18:41:13.201356Z","shell.execute_reply":"2022-03-06T18:43:00.553506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(\n    X_train,\n    y_train,\n    validation_split=0.1,\n    batch_size=512,\n    epochs=10\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-06T18:46:21.808572Z","iopub.execute_input":"2022-03-06T18:46:21.808855Z","iopub.status.idle":"2022-03-06T18:49:51.813416Z","shell.execute_reply.started":"2022-03-06T18:46:21.808804Z","shell.execute_reply":"2022-03-06T18:49:51.812724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-06T18:44:14.899446Z","iopub.execute_input":"2022-03-06T18:44:14.899699Z","iopub.status.idle":"2022-03-06T18:44:14.905061Z","shell.execute_reply.started":"2022-03-06T18:44:14.89967Z","shell.execute_reply":"2022-03-06T18:44:14.903864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow\ntensorflow.config.experimental.list_physical_devices()","metadata":{"execution":{"iopub.status.busy":"2022-03-06T18:40:33.145758Z","iopub.execute_input":"2022-03-06T18:40:33.146506Z","iopub.status.idle":"2022-03-06T18:40:33.152247Z","shell.execute_reply.started":"2022-03-06T18:40:33.146469Z","shell.execute_reply":"2022-03-06T18:40:33.151529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(h.history['loss'], label='Train')\nplt.plot(h.history['val_loss'], label='Val')\nplt.legend();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate","metadata":{}},{"cell_type":"code","source":"def evaluate(model, X, y):\n    probs = model.predict(X)\n#     preds = np.argmax(probs, axis=1)\n#     report = metrics.classification_report(y, preds)\n#     print(report)\n    df_sub = pd.DataFrame({\n        'away': probs[:, 0],\n        'draw': probs[:, 1],\n        'home': probs[:, 2]\n    })\n    df_sub['draw']=np.where(df_sub['draw']>0.38,0.95,0)\n    df_sub['away']=np.where(df_sub['draw']==0.95,0.025,df_sub['away'])\n    df_sub['home']=np.where(df_sub['draw']==0.95,0.025,df_sub['home'])\n    probs=df_sub.values\n    logloss = metrics.log_loss(y, probs)\n    print('Log loss:', logloss)\n#     cm = metrics.confusion_matrix(y, preds)\n#     disp = metrics.ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=list(target2int.keys()))\n#     disp.plot(cmap=plt.cm.Blues)\n#     plt.show();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate(model, X_val, y_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # probs = model.predict(X_val)\n# df_sub = pd.DataFrame({\n#     'away': probs[:, 0],\n#     'draw': probs[:, 1],\n#     'home': probs[:, 2]\n# })\n# df_sub['draw']=np.where(df_sub['draw']>0.35,1,0)\n# df_sub['away']=np.where(df_sub['draw']==1,0,df_sub['away'])\n# df_sub['home']=np.where(df_sub['draw']==1,0,df_sub['home'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems that the `draw` class is hurting the log loss.   ","metadata":{}},{"cell_type":"markdown","source":"## Predict on test set","metadata":{}},{"cell_type":"code","source":"# away, draw, home\nprobs_test = model.predict(X_test)\nprobs_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.concat([\n#     train['target_int'].value_counts(normalize=True).sort_index().rename('observed'),\n#     pd.Series(np.argmax(df_sub.iloc[:, 1:].values, axis=1), name='predicted').value_counts(normalize=True).sort_index()\n# ], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.DataFrame({\n    'id': test['id'],\n    'away': probs_test[:, 0],\n    'draw': probs_test[:, 1],\n    'home': probs_test[:, 2]\n})\ndf_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Very few predicted draws compared to the observed target.   ","metadata":{}},{"cell_type":"code","source":"df_sub.iloc[:, 1:].plot.hist(alpha=0.5);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission expects this order: id, home, away, draw\ndf_sub[['id', 'home', 'away', 'draw']].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-03-05T03:17:42.08742Z","iopub.execute_input":"2022-03-05T03:17:42.087746Z","iopub.status.idle":"2022-03-05T03:17:42.871517Z","shell.execute_reply.started":"2022-03-05T03:17:42.087705Z","shell.execute_reply":"2022-03-05T03:17:42.870329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-03-05T02:55:00.472136Z","iopub.status.idle":"2022-03-05T02:55:00.472585Z","shell.execute_reply.started":"2022-03-05T02:55:00.472384Z","shell.execute_reply":"2022-03-05T02:55:00.472406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}