{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Football Match Probability Prediction**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nimport time\nwarnings.simplefilter('ignore')\nimport math\nfrom statistics import mean\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib import style\nsubmit = True\nsearch = False\nestimators = 20\nfrom datetime import timedelta\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import log_loss","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:17.649459Z","iopub.execute_input":"2022-05-16T13:03:17.650666Z","iopub.status.idle":"2022-05-16T13:03:19.120371Z","shell.execute_reply.started":"2022-05-16T13:03:17.650503Z","shell.execute_reply":"2022-05-16T13:03:19.119358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/football-match-probability-prediction/train.csv')\nX_test = pd.read_csv('../input/football-match-probability-prediction/test.csv')\ntrain_target_score = pd.read_csv('../input/football-match-probability-prediction/train_target_and_scores.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:19.122160Z","iopub.execute_input":"2022-05-16T13:03:19.122435Z","iopub.status.idle":"2022-05-16T13:03:28.711310Z","shell.execute_reply.started":"2022-05-16T13:03:19.122378Z","shell.execute_reply":"2022-05-16T13:03:28.710412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:28.712524Z","iopub.execute_input":"2022-05-16T13:03:28.712754Z","iopub.status.idle":"2022-05-16T13:03:28.759741Z","shell.execute_reply.started":"2022-05-16T13:03:28.712727Z","shell.execute_reply":"2022-05-16T13:03:28.758903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:28.762310Z","iopub.execute_input":"2022-05-16T13:03:28.763580Z","iopub.status.idle":"2022-05-16T13:03:28.935313Z","shell.execute_reply.started":"2022-05-16T13:03:28.763515Z","shell.execute_reply":"2022-05-16T13:03:28.934390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:28.936674Z","iopub.execute_input":"2022-05-16T13:03:28.937017Z","iopub.status.idle":"2022-05-16T13:03:30.482483Z","shell.execute_reply.started":"2022-05-16T13:03:28.936987Z","shell.execute_reply":"2022-05-16T13:03:30.481460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_columns = 'home_team_history_coach'\n###\ncols = []\nfor col in train.filter(regex=target_columns, axis=1).columns:\n    cols.append(col)\nprint(cols)\ncorr = pd.concat([train[cols],train_target_score], axis=1).corr()\nsns.heatmap(corr)","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:30.485306Z","iopub.execute_input":"2022-05-16T13:03:30.485607Z","iopub.status.idle":"2022-05-16T13:03:31.002325Z","shell.execute_reply.started":"2022-05-16T13:03:30.485577Z","shell.execute_reply":"2022-05-16T13:03:31.001217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_columns = 'away_team_history_coach'\n###\ncols = []\nfor col in train.filter(regex=target_columns, axis=1).columns:\n    cols.append(col)\nprint(cols)\ncorr = pd.concat([train[cols],train_target_score], axis=1).corr()\nsns.heatmap(corr)","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:31.003795Z","iopub.execute_input":"2022-05-16T13:03:31.004474Z","iopub.status.idle":"2022-05-16T13:03:31.471761Z","shell.execute_reply.started":"2022-05-16T13:03:31.004434Z","shell.execute_reply":"2022-05-16T13:03:31.470836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:31.474176Z","iopub.execute_input":"2022-05-16T13:03:31.474420Z","iopub.status.idle":"2022-05-16T13:03:31.496170Z","shell.execute_reply.started":"2022-05-16T13:03:31.474380Z","shell.execute_reply":"2022-05-16T13:03:31.495256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_streak(train):\n    h_t_goal = \"home_team_history_goal_\"\n    h_t_opp_goal = \"home_team_history_opponent_goal_\"\n    a_t_goal = \"away_team_history_goal_\"\n    a_t_opp_goal = \"away_team_history_opponent_goal_\"\n    train[\"away_streak\"] = np.nan\n    train[\"home_streak\"] = np.nan\n    def apply_streaks(x):\n        home_streak = 0\n        away_streak = 0\n        home_streak_over = False\n        away_streak_over = False\n        \n        if  x[h_t_goal+str(1)] > x[h_t_opp_goal+str(1)]:\n            home_win_streak = True\n        else:\n            home_win_streak = False\n            \n        if  x[a_t_goal+str(1)] > x[a_t_opp_goal+str(1)]:\n            away_win_streak = True\n        else:\n            away_win_streak = False\n            \n        def check_streak(streak,streak_over,win_streak,team_goal,opp_goal):\n            if not streak_over and win_streak and team_goal > opp_goal:\n                streak+=1\n            elif not streak_over and not win_streak and team_goal < opp_goal:\n                streak-=1\n            elif not streak_over and team_goal == opp_goal:\n                pass\n            else:\n                streak_over = True\n            return streak, streak_over\n                \n        for i in range(1,11):\n            if not home_streak_over:\n                home_streak, home_streak_over = check_streak(home_streak,home_streak_over, home_win_streak, x[h_t_goal+str(i)],x[h_t_opp_goal+str(i)])\n            if not away_streak_over:\n                away_streak, away_streak_over = check_streak(away_streak,away_streak_over, away_win_streak, x[a_t_goal+str(i)],x[a_t_opp_goal+str(i)])\n            \n        return pd.Series([home_streak,away_streak],index=['home_streak','away_streak'])\n\n    \n    train[['home_streak','away_streak']] = train.apply(lambda x: apply_streaks(x), axis=1)\n    return train","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:31.497512Z","iopub.execute_input":"2022-05-16T13:03:31.497747Z","iopub.status.idle":"2022-05-16T13:03:31.510536Z","shell.execute_reply.started":"2022-05-16T13:03:31.497719Z","shell.execute_reply":"2022-05-16T13:03:31.509439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import category_encoders as ce\nfrom sklearn.model_selection import train_test_split\n\n\nX_train = train.drop(['target'], axis=1)\ny_train = train['target']\n\nX_train = add_streak(X_train)\nX_test = add_streak(X_test)\n\nif not submit:\n    X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size = 0.2, random_state = 42)\n\nencoder = ce.OrdinalEncoder(cols=['home_team_name' , 'away_team_name', 'is_cup'])\nX_train = encoder.fit_transform(X_train)\nX_test = encoder.transform(X_test)\ntemp_df = X_test\nfor i in X_train.columns:\n    if X_train[i].dtype == 'object':\n        X_train = X_train.drop([i],axis=1)\n        X_test = X_test.drop([i],axis=1)\n    if \"is_cup\" in i or \"history_is_play_home\" in i:\n        try:\n            X_train = X_train.drop([i],axis=1)\n            X_test = X_test.drop([i],axis=1)\n        except:\n            continue\n\nfor i in X_train.columns:\n    if 'rating' not in i and 'goal':\n        X_train[i] = X_train[i].fillna(-1)\n        X_test[i] = X_test[i].fillna(-1)\n    else:\n        X_train[i] = X_train[i].fillna(X_train[i].mode())\n        X_test[i] = X_test[i].fillna(X_train[i].mode())\nlag_size = 9\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:03:31.513811Z","iopub.execute_input":"2022-05-16T13:03:31.514051Z","iopub.status.idle":"2022-05-16T13:04:50.528711Z","shell.execute_reply.started":"2022-05-16T13:03:31.514024Z","shell.execute_reply":"2022-05-16T13:04:50.527198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in X_train.columns:\n    print(i)","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:04:50.530414Z","iopub.execute_input":"2022-05-16T13:04:50.530658Z","iopub.status.idle":"2022-05-16T13:04:50.556816Z","shell.execute_reply.started":"2022-05-16T13:04:50.530631Z","shell.execute_reply":"2022-05-16T13:04:50.555855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import log_loss,accuracy_score\nif not search:\n    best_hyperparams = {'colsample_bytree': 0.9080289013373745, 'gamma': 1.2419885617337658, 'max_depth': 8.0, 'min_child_weight': 4.0, 'n_estimators': 238.0, 'reg_alpha': 41.0, 'reg_lambda': 0.05050398016408508}\nif not submit and not search:\n    rfc = XGBClassifier(n_estimators=int(best_hyperparams['n_estimators']), max_depth = int(best_hyperparams['max_depth']), gamma = best_hyperparams['gamma'],\n                    reg_alpha = int(best_hyperparams['reg_alpha']),min_child_weight=int(best_hyperparams['min_child_weight']),\n                    colsample_bytree=int(best_hyperparams['colsample_bytree']))\n    rfc.fit(X_train, y_train)\n    y_pred = rfc.predict_proba(X_test)\n    y_pred_acc = rfc.predict(X_test)\n\n    print(y_test.values)\n    print(y_pred)\n    print('Log loss score with  : {0:0.4f}'. format(log_loss(y_test.values,y_pred)))\n    print('Model accuracy score with : {0:0.4f}'. format(accuracy_score(y_test, y_pred_acc)))","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:04:50.558275Z","iopub.execute_input":"2022-05-16T13:04:50.558596Z","iopub.status.idle":"2022-05-16T13:04:50.893328Z","shell.execute_reply.started":"2022-05-16T13:04:50.558553Z","shell.execute_reply":"2022-05-16T13:04:50.892429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not submit and not search:\n    feature_scores = pd.Series(rfc.feature_importances_, index=X_train.columns).sort_values(ascending=False)\n\n    for i,v in feature_scores.items():\n        print(i, v)","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:04:50.894642Z","iopub.execute_input":"2022-05-16T13:04:50.894902Z","iopub.status.idle":"2022-05-16T13:04:50.899919Z","shell.execute_reply.started":"2022-05-16T13:04:50.894872Z","shell.execute_reply":"2022-05-16T13:04:50.899040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from hyperopt import hp\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\n\nif search:\n\n    space={'max_depth': hp.quniform(\"max_depth\", 3, 18, 1),\n            'gamma': hp.uniform ('gamma', 1,9),\n            'reg_alpha' : hp.quniform('reg_alpha', 40,180,1),\n            'reg_lambda' : hp.uniform('reg_lambda', 0,1),\n            'colsample_bytree' : hp.uniform('colsample_bytree', 0.5,1),\n            'min_child_weight' : hp.quniform('min_child_weight', 0, 10, 1),\n            'n_estimators': hp.quniform('n_estimators', 150, 250, 1),\n            'seed': 0\n        }\n\n    def objective(space):\n        clf=xgb.XGBClassifier(\n                        n_estimators =int(space['n_estimators']), max_depth = int(space['max_depth']), gamma = space['gamma'],\n                        reg_alpha = int(space['reg_alpha']),min_child_weight=int(space['min_child_weight']),\n                        colsample_bytree=int(space['colsample_bytree']))\n\n        evaluation = [( X_train, y_train), ( X_test, y_test)]\n\n        clf.fit(X_train, y_train,\n                eval_set=evaluation, eval_metric=\"mlogloss\",\n                early_stopping_rounds=100,verbose=False)\n\n\n        y_pred = clf.predict_proba(X_test)\n        accuracy = log_loss(y_test.values,y_pred)\n        print (\"SCORE:\", accuracy)\n        return {'loss': accuracy, 'status': STATUS_OK }\n\n    trials = Trials()\n\n    best_hyperparams = fmin(fn = objective,\n                            space = space,\n                            algo = tpe.suggest,\n                            max_evals = 100,\n                            trials = trials)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:04:50.901276Z","iopub.execute_input":"2022-05-16T13:04:50.901568Z","iopub.status.idle":"2022-05-16T13:04:51.471711Z","shell.execute_reply.started":"2022-05-16T13:04:50.901523Z","shell.execute_reply":"2022-05-16T13:04:51.470339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    print(best_hyperparams)","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:04:51.473655Z","iopub.execute_input":"2022-05-16T13:04:51.474042Z","iopub.status.idle":"2022-05-16T13:04:51.481765Z","shell.execute_reply.started":"2022-05-16T13:04:51.473998Z","shell.execute_reply":"2022-05-16T13:04:51.480199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nif submit:\n    pipeline = Pipeline(steps=[\n        (\"classifier\", XGBClassifier(n_estimators=int(best_hyperparams['n_estimators']), max_depth = int(best_hyperparams['max_depth']), gamma = best_hyperparams['gamma'],\n                    reg_alpha = int(best_hyperparams['reg_alpha']),min_child_weight=int(best_hyperparams['min_child_weight']),\n                    colsample_bytree=int(best_hyperparams['colsample_bytree'])))])\n\n    pipeline.fit(X_train, y_train)\n    predictions = pipeline.predict_proba(X_test)\n    submission = pd.DataFrame(predictions ,columns=pipeline.classes_)\n\n    #do not forget the id column\n    submission['id'] = temp_df['id']\n    display(submission)\n    #submit!\n    submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-16T13:04:51.484094Z","iopub.execute_input":"2022-05-16T13:04:51.484508Z","iopub.status.idle":"2022-05-16T13:05:23.762270Z","shell.execute_reply.started":"2022-05-16T13:04:51.484455Z","shell.execute_reply":"2022-05-16T13:05:23.761085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}