{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\nsubmit = True\nsearch = False\nestimators = 20\ndf = pd.read_csv(\"../input/football-match-probability-prediction/train.csv\")\nX_test = pd.read_csv(\"../input/football-match-probability-prediction/test.csv\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-22T10:44:57.634822Z","iopub.execute_input":"2022-03-22T10:44:57.635105Z","iopub.status.idle":"2022-03-22T10:45:04.736842Z","shell.execute_reply.started":"2022-03-22T10:44:57.635076Z","shell.execute_reply":"2022-03-22T10:45:04.735714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndisplay(df.head())\ndf.shape\nfor i in df.columns:\n    print(i)","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:45:04.738510Z","iopub.execute_input":"2022-03-22T10:45:04.738761Z","iopub.status.idle":"2022-03-22T10:45:04.789610Z","shell.execute_reply.started":"2022-03-22T10:45:04.738731Z","shell.execute_reply":"2022-03-22T10:45:04.783866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:45:04.797727Z","iopub.execute_input":"2022-03-22T10:45:04.797996Z","iopub.status.idle":"2022-03-22T10:45:04.818094Z","shell.execute_reply.started":"2022-03-22T10:45:04.797962Z","shell.execute_reply":"2022-03-22T10:45:04.817379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_points(df):\n    h_t_goal = \"home_team_history_goal_\"\n    h_t_opp_goal = \"home_team_history_opponent_goal_\"\n    a_t_goal = \"away_team_history_goal_\"\n    a_t_opp_goal = \"away_team_history_opponent_goal_\"\n    for i in range(1,11):\n        df[\"home_team_history_point_\" + str(i)] = np.nan\n        df['away_team_history_point_' + str(i)] = np.nan\n    def apply_points(x,idx):\n\n        def check_points(team_goal,opp_goal):\n            if team_goal > opp_goal:\n                return 3\n            elif team_goal < opp_goal:\n                return 0\n            elif team_goal == opp_goal:\n                return 1\n                \n        for i in range(1,11):\n            df.at[idx,\"home_team_history_point_\" + str(i)] = check_points(x[h_t_goal+str(i)],x[h_t_opp_goal+str(i)])\n            df.at[idx,\"away_team_history_point_\" + str(i)] = check_points(x[a_t_goal+str(i)],x[a_t_opp_goal+str(i)])\n            \n        #return pd.Series([home_streak,home_streak_type,away_streak,away_streak_type],index=['home_streak','home_streak_type','away_streak','away_streak_type'])\n\n    \n    #df[['home_streak','home_streak_type','away_streak','away_streak_type']] = df.apply(lambda x: apply_streaks(x,x.name), axis=1)\n    df.apply(lambda x: apply_points(x,x.name), axis=1)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:45:04.820851Z","iopub.execute_input":"2022-03-22T10:45:04.821114Z","iopub.status.idle":"2022-03-22T10:45:04.834544Z","shell.execute_reply.started":"2022-03-22T10:45:04.821082Z","shell.execute_reply":"2022-03-22T10:45:04.833418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_streak(df):\n    h_t_goal = \"home_team_history_goal_\"\n    h_t_opp_goal = \"home_team_history_opponent_goal_\"\n    a_t_goal = \"away_team_history_goal_\"\n    a_t_opp_goal = \"away_team_history_opponent_goal_\"\n    df[\"away_streak\"] = np.nan\n    df[\"home_streak\"] = np.nan\n    def apply_streaks(x):\n        home_streak = 0\n        away_streak = 0\n        home_streak_over = False\n        away_streak_over = False\n        \n        if  x[h_t_goal+str(1)] > x[h_t_opp_goal+str(1)]:\n            home_win_streak = True\n        else:\n            home_win_streak = False\n            \n        if  x[a_t_goal+str(1)] > x[a_t_opp_goal+str(1)]:\n            away_win_streak = True\n        else:\n            away_win_streak = False\n            \n        def check_streak(streak,streak_over,win_streak,team_goal,opp_goal):\n            if not streak_over and win_streak and team_goal > opp_goal:\n                streak+=1\n            elif not streak_over and not win_streak and team_goal < opp_goal:\n                streak-=1\n            elif not streak_over and team_goal == opp_goal:\n                pass\n            else:\n                streak_over = True\n            return streak, streak_over\n                \n        for i in range(1,11):\n            if not home_streak_over:\n                home_streak, home_streak_over = check_streak(home_streak,home_streak_over, home_win_streak, x[h_t_goal+str(i)],x[h_t_opp_goal+str(i)])\n            if not away_streak_over:\n                away_streak, away_streak_over = check_streak(away_streak,away_streak_over, away_win_streak, x[a_t_goal+str(i)],x[a_t_opp_goal+str(i)])\n            \n        return pd.Series([home_streak,away_streak],index=['home_streak','away_streak'])\n\n    \n    df[['home_streak','away_streak']] = df.apply(lambda x: apply_streaks(x), axis=1)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:45:04.836210Z","iopub.execute_input":"2022-03-22T10:45:04.836486Z","iopub.status.idle":"2022-03-22T10:45:04.859076Z","shell.execute_reply.started":"2022-03-22T10:45:04.836454Z","shell.execute_reply":"2022-03-22T10:45:04.858205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_total_points(df, home_or_away='home'):\n    cols = []\n    for i in df.columns:\n        if \"point\" in i and home_or_away in i:\n            cols.append(i)\n    df[home_or_away + '_points_last_10'] = df[cols].sum(axis=1)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:45:04.860362Z","iopub.execute_input":"2022-03-22T10:45:04.860599Z","iopub.status.idle":"2022-03-22T10:45:04.879740Z","shell.execute_reply.started":"2022-03-22T10:45:04.860570Z","shell.execute_reply":"2022-03-22T10:45:04.878920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import category_encoders as ce\nfrom sklearn.model_selection import train_test_split\n\n\nX_train = df.drop(['target'], axis=1)\ny_train = df['target']\n\nX_train = add_streak(X_train)\nX_test = add_streak(X_test)\n\nX_train = add_points(X_train)\nX_test = add_points(X_test)\n\nX_train = add_total_points(X_train,home_or_away='home')\nX_test = add_total_points(X_test,home_or_away='home')\nX_train = add_total_points(X_train,home_or_away='away')\nX_test = add_total_points(X_test,home_or_away='away')\n\n\nif not submit:\n    X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size = 0.2, random_state = 42)\n\nencoder = ce.OrdinalEncoder(cols=['home_team_name' , 'away_team_name', 'is_cup'])\nX_train = encoder.fit_transform(X_train)\nX_test = encoder.transform(X_test)\ntemp_df = X_test\nfor i in X_train.columns:\n    if X_train[i].dtype == 'object':\n        X_train = X_train.drop([i],axis=1)\n        X_test = X_test.drop([i],axis=1)\n    if \"is_cup\" in i or \"history_is_play_home\" in i or \"name\" in i or \"history_point\" in i:\n        try:\n            X_train = X_train.drop([i],axis=1)\n            X_test = X_test.drop([i],axis=1)\n        except:\n            continue\n\nfor i in X_train.columns:\n    if 'rating' not in i and 'goal' not in i:\n        X_train[i] = X_train[i].fillna(-1)\n        X_test[i] = X_test[i].fillna(-1)\n    else:\n        X_train[i] = X_train[i].fillna(X_train[i].mode())\n        X_test[i] = X_test[i].fillna(X_train[i].mode())\nlag_size = 9\n#for i in X_train.columns:\n#    if sum([1 for num in range(lag_size,11)  if str(num) in i]) >= 1:\n#        X_train = X_train.drop([i],axis=1)\n#        X_test = X_test.drop([i],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:45:04.881438Z","iopub.execute_input":"2022-03-22T10:45:04.881670Z","iopub.status.idle":"2022-03-22T10:48:19.446083Z","shell.execute_reply.started":"2022-03-22T10:45:04.881643Z","shell.execute_reply":"2022-03-22T10:48:19.445068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train.head(25)\n\"\"\"X_train[['home_streak','away_streak']].head(20)\nh_t_goal = \"away_team_history_goal_\"\nh_t_opp_goal = \"away_team_history_opponent_goal_\"\ncols = []\ncols.append('away_streak')\nfor i in range(1,5):\n    cols.append(h_t_goal+str(i))\n    cols.append(h_t_opp_goal+str(i))\nX_train[cols].head(20)\"\"\"\n\nfor i in X_train.columns:\n    print(i)","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:48:19.447514Z","iopub.execute_input":"2022-03-22T10:48:19.447751Z","iopub.status.idle":"2022-03-22T10:48:19.475901Z","shell.execute_reply.started":"2022-03-22T10:48:19.447724Z","shell.execute_reply":"2022-03-22T10:48:19.474899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import log_loss,accuracy_score\nif not search:\n    best_hyperparams = {'colsample_bytree': 0.9080289013373745, 'gamma': 1.2419885617337658, 'max_depth': 8.0, 'min_child_weight': 4.0, 'n_estimators': 238.0, 'reg_alpha': 41.0, 'reg_lambda': 0.05050398016408508}\nif not submit and not search:\n    rfc = XGBClassifier(n_estimators=int(best_hyperparams['n_estimators']), max_depth = int(best_hyperparams['max_depth']), gamma = best_hyperparams['gamma'],\n                    reg_alpha = int(best_hyperparams['reg_alpha']),min_child_weight=int(best_hyperparams['min_child_weight']),\n                    colsample_bytree=int(best_hyperparams['colsample_bytree']))\n    #fc = XGBClassifier(n_estimators=estimators)\n    rfc.fit(X_train, y_train)\n    y_pred = rfc.predict_proba(X_test)\n    y_pred_acc = rfc.predict(X_test)\n\n    print(y_test.values)\n    print(y_pred)\n    print('Log loss score with  : {0:0.4f}'. format(log_loss(y_test.values,y_pred)))\n    print('Model accuracy score with : {0:0.4f}'. format(accuracy_score(y_test, y_pred_acc)))","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:48:19.477184Z","iopub.execute_input":"2022-03-22T10:48:19.477430Z","iopub.status.idle":"2022-03-22T10:48:45.394721Z","shell.execute_reply.started":"2022-03-22T10:48:19.477402Z","shell.execute_reply":"2022-03-22T10:48:45.393847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not submit and not search:\n    feature_scores = pd.Series(rfc.feature_importances_, index=X_train.columns).sort_values(ascending=False)\n\n    for i,v in feature_scores.items():\n        print(i, v)","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:48:45.397720Z","iopub.execute_input":"2022-03-22T10:48:45.398314Z","iopub.status.idle":"2022-03-22T10:48:45.438474Z","shell.execute_reply.started":"2022-03-22T10:48:45.398240Z","shell.execute_reply":"2022-03-22T10:48:45.437420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/prashant111/a-guide-on-xgboost-hyperparameters-tuning\nfrom hyperopt import hp\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\n\nif search:\n\n    space={'max_depth': hp.quniform(\"max_depth\", 3, 18, 1),\n            'gamma': hp.uniform ('gamma', 1,9),\n            'reg_alpha' : hp.quniform('reg_alpha', 40,180,1),\n            'reg_lambda' : hp.uniform('reg_lambda', 0,1),\n            'colsample_bytree' : hp.uniform('colsample_bytree', 0.5,1),\n            'min_child_weight' : hp.quniform('min_child_weight', 0, 10, 1),\n            'n_estimators': hp.quniform('n_estimators', 150, 250, 1),\n            'seed': 0\n        }\n\n    def objective(space):\n        clf=xgb.XGBClassifier(\n                        n_estimators =int(space['n_estimators']), max_depth = int(space['max_depth']), gamma = space['gamma'],\n                        reg_alpha = int(space['reg_alpha']),min_child_weight=int(space['min_child_weight']),\n                        colsample_bytree=int(space['colsample_bytree']))\n\n        evaluation = [( X_train, y_train), ( X_test, y_test)]\n\n        clf.fit(X_train, y_train,\n                eval_set=evaluation, eval_metric=\"mlogloss\",\n                early_stopping_rounds=100,verbose=False)\n\n\n        y_pred = clf.predict_proba(X_test)\n        accuracy = log_loss(y_test.values,y_pred)\n        print (\"SCORE:\", accuracy)\n        return {'loss': accuracy, 'status': STATUS_OK }\n\n    trials = Trials()\n\n    best_hyperparams = fmin(fn = objective,\n                            space = space,\n                            algo = tpe.suggest,\n                            max_evals = 100,\n                            trials = trials)","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:48:45.440068Z","iopub.execute_input":"2022-03-22T10:48:45.440410Z","iopub.status.idle":"2022-03-22T10:48:45.455033Z","shell.execute_reply.started":"2022-03-22T10:48:45.440367Z","shell.execute_reply":"2022-03-22T10:48:45.454012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(best_hyperparams)","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:48:45.456708Z","iopub.execute_input":"2022-03-22T10:48:45.457009Z","iopub.status.idle":"2022-03-22T10:48:45.474264Z","shell.execute_reply.started":"2022-03-22T10:48:45.456968Z","shell.execute_reply":"2022-03-22T10:48:45.473108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nif submit:\n    pipeline = Pipeline(steps=[\n        (\"classifier\", XGBClassifier(n_estimators=int(best_hyperparams['n_estimators']), max_depth = int(best_hyperparams['max_depth']), gamma = best_hyperparams['gamma'],\n                    reg_alpha = int(best_hyperparams['reg_alpha']),min_child_weight=int(best_hyperparams['min_child_weight']),\n                    colsample_bytree=int(best_hyperparams['colsample_bytree'])))])\n\n    pipeline.fit(X_train, y_train)\n    predictions = pipeline.predict_proba(X_test)\n    submission = pd.DataFrame(predictions ,columns=pipeline.classes_)\n\n    #do not forget the id column\n    submission['id'] = temp_df['id']\n    display(submission)\n    #submit!\n    submission.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-22T10:48:45.475726Z","iopub.execute_input":"2022-03-22T10:48:45.476038Z","iopub.status.idle":"2022-03-22T10:48:45.488161Z","shell.execute_reply.started":"2022-03-22T10:48:45.475996Z","shell.execute_reply":"2022-03-22T10:48:45.487303Z"},"trusted":true},"execution_count":null,"outputs":[]}]}