{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os, json, re\nimport numpy as np\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-09T15:54:44.281891Z","iopub.execute_input":"2023-11-09T15:54:44.282343Z","iopub.status.idle":"2023-11-09T15:54:44.290192Z","shell.execute_reply.started":"2023-11-09T15:54:44.282302Z","shell.execute_reply":"2023-11-09T15:54:44.288515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install catboost\n!pip install xgboost\nimport xgboost as xgb\nimport catboost as cb","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:54:45.711081Z","iopub.execute_input":"2023-11-09T15:54:45.711558Z","iopub.status.idle":"2023-11-09T15:55:14.742602Z","shell.execute_reply.started":"2023-11-09T15:54:45.711496Z","shell.execute_reply":"2023-11-09T15:55:14.740681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/football-match-probability-prediction/train.csv\")\ndf.fillna(\"\", inplace=True)\nprint(len(df))","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:55:14.745087Z","iopub.execute_input":"2023-11-09T15:55:14.745500Z","iopub.status.idle":"2023-11-09T15:55:21.991339Z","shell.execute_reply.started":"2023-11-09T15:55:14.745459Z","shell.execute_reply":"2023-11-09T15:55:21.989985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2 = pd.read_csv(\"/kaggle/input/football-match-probability-prediction/test.csv\")\ndf2.fillna(\"\", inplace=True)\nprint(len(df2))","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:55:21.992728Z","iopub.execute_input":"2023-11-09T15:55:21.993122Z","iopub.status.idle":"2023-11-09T15:55:26.324825Z","shell.execute_reply.started":"2023-11-09T15:55:21.993090Z","shell.execute_reply":"2023-11-09T15:55:26.323568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_things(df, phase='train'):\n    \n    if phase == 'train':\n        #If do not have value of day 1, then just delete the row\n        df = df[df['home_team_history_match_date_1'] != \"\"]\n        # There is just 1 row is_cup == \"\", so delete this row\n        df = df[df['is_cup'] != \"\"]\n    # in inference stage, there is no team name, so I remove this feature from beginning\n    del df['home_team_name']\n    del df['away_team_name']\n    # there are so many NaN in home(away)_team_coach_id, so I decide to ignore these columns\n    del df['home_team_coach_id']\n    del df['away_team_coach_id']\n    # I remove league_name too, since len(set(df2['league_name'].unique()) - set(df['league_name'].unique())) > 0\n    del df['league_name']\n    # I remove league_id too, since len(set(df2['league_id'].unique()) - set(df['league_id'].unique())) > 0\n    del df['league_id']\n    \n    print(len(df))\n    return df\ndf = remove_things(df, \"train\")\ndf2 = remove_things(df2, \"test\")","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:55:26.327598Z","iopub.execute_input":"2023-11-09T15:55:26.327997Z","iopub.status.idle":"2023-11-09T15:55:27.926488Z","shell.execute_reply.started":"2023-11-09T15:55:26.327965Z","shell.execute_reply":"2023-11-09T15:55:27.925476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def processing(df, phase=\"train\"):\n    # Convert date to datetime obj\n    for col in df.filter(regex='date', axis=1).columns:\n        df[col] = pd.to_datetime(df[col])\n    df['is_cup'] = df['is_cup'].astype('int32')\n    func_to_int = lambda x: int(x) if x != \"\" else np.nan\n    func_to_float = lambda x: float(x) if x != \"\" else np.nan\n    for i in range(1, 11):\n        for team in [\"home\", \"away\"]:\n            for col in [f'{team}_team_history_is_play_home_{i}',\n                        f'{team}_team_history_is_cup_{i}',\n                        f'{team}_team_history_goal_{i}',\n                        f'{team}_team_history_opponent_goal_{i}',\n                        \n                       ]:\n                df[col] = df[col].apply(func_to_int)\n            for col in [f'{team}_team_history_rating_{i}',\n                        \n                       ]:\n                df[col] = df[col].apply(func_to_float)\n    def norm_coach(values):\n        norm_values = []\n        value_to_norm_value = {}\n        max_norm_value = -1\n        for value in values:\n            if value == \"\":\n                norm_values.append(np.nan)\n            else:\n                if value in value_to_norm_value:\n                    norm_values.append(value_to_norm_value[value])\n                else:\n                    max_norm_value += 1\n                    value_to_norm_value[value] = max_norm_value\n                    norm_values.append(int(value_to_norm_value[value]))\n        return pd.Series(norm_values)\n\n    for team in [\"home\", \"away\"]:\n        norm_coach_columns = [f'{team}_change_coach_{i}' for i in range(1, 11)]\n        coach_columns = [f'{team}_team_history_coach_{i}' for i in range(1, 11)]\n        df[norm_coach_columns] = df[coach_columns].apply(lambda x: norm_coach(x.values), axis=1)\n        for col in coach_columns:\n            del df[col]\n    print(len(df))\n    return df\ndf  = processing(df, \"train\")\ndf2 = processing(df2, \"test\")\n","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:55:27.928569Z","iopub.execute_input":"2023-11-09T15:55:27.929048Z","iopub.status.idle":"2023-11-09T15:57:09.605108Z","shell.execute_reply.started":"2023-11-09T15:55:27.929013Z","shell.execute_reply":"2023-11-09T15:57:09.603700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_time_features(df):\n#     func_get_hour = lambda x: x.hour\n#     func_get_weekday = lambda x: x.dayofweek\n#     df['match_hour'] = df['match_date'].apply(func_get_hour)\n#     df['match_weekday'] = df['match_date'].apply(func_get_weekday)\n#     for i in range(1, 11):\n#         for team in ['home', 'away']:\n#             col = f'{team}_team_history_match_date_{i}'\n#             new_col = f'{team}_team_history_match_hour_{i}'\n#             df[new_col] = df[col].apply(func_get_hour)\n#             new_col = f'{team}_team_history_match_weekday_{i}'\n#             df[new_col] = df[col].apply(func_get_weekday)\n#     return df\n# df = get_time_features(df)\n# df2 = get_time_features(df2)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:14:54.966012Z","iopub.execute_input":"2023-11-09T15:14:54.966402Z","iopub.status.idle":"2023-11-09T15:15:24.350730Z","shell.execute_reply.started":"2023-11-09T15:14:54.966371Z","shell.execute_reply":"2023-11-09T15:15:24.348846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove2(df):\n    for col in df.columns:\n        if \"match_date\" in col:\n            del df[col]\n    ids = df['id']\n    del df['id']\n    print(len(df))\n    return df, ids\ndf, _ = remove2(df)\ndf2, df2_ids = remove2(df2)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:57:09.606541Z","iopub.execute_input":"2023-11-09T15:57:09.606889Z","iopub.status.idle":"2023-11-09T15:57:09.730947Z","shell.execute_reply.started":"2023-11-09T15:57:09.606860Z","shell.execute_reply":"2023-11-09T15:57:09.729295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = df['target']\nX_train = df.drop(['target'], axis=1)\nX_test = df2","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:57:09.732453Z","iopub.execute_input":"2023-11-09T15:57:09.732947Z","iopub.status.idle":"2023-11-09T15:57:09.986609Z","shell.execute_reply.started":"2023-11-09T15:57:09.732899Z","shell.execute_reply":"2023-11-09T15:57:09.985655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nclasses = ['home', 'draw', 'away']\n# df_train, df_valid = train_test_split(df, train_size=0.8, random_state=42)\n# y_train = df_train['target'].apply(lambda x: classes.index(x))\n# X_train = df_train.drop(['target'], axis=1)\n\n# y_valid = df_valid['target'].apply(lambda x: classes.index(x))\n# X_valid = df_valid.drop(['target'], axis=1)\n\ny_train = df['target'].apply(lambda x: classes.index(x))\nX_train = df.drop(['target'], axis=1)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:57:09.988441Z","iopub.execute_input":"2023-11-09T15:57:09.989504Z","iopub.status.idle":"2023-11-09T15:57:10.269054Z","shell.execute_reply.started":"2023-11-09T15:57:09.989457Z","shell.execute_reply":"2023-11-09T15:57:10.267734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna","metadata":{"execution":{"iopub.status.busy":"2023-11-06T06:41:19.620851Z","iopub.execute_input":"2023-11-06T06:41:19.621320Z","iopub.status.idle":"2023-11-06T06:41:20.976350Z","shell.execute_reply.started":"2023-11-06T06:41:19.621279Z","shell.execute_reply":"2023-11-06T06:41:20.975197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier\nmodel = CatBoostClassifier(\n    iterations=1000,\n    learning_rate=0.1,\n)\nmodel.fit(X_train, y_train, \n          #eval_set=(X_valid, y_valid), \n          verbose=True\n         )\n\n# import optuna\n# from sklearn.metrics import accuracy_score\n# def objective(trial):\n#     param = {\n# #         \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.01, 0.1),\n# #         \"depth\": trial.suggest_int(\"depth\", 1, 12),\n#         \"iterations\": 100,\n#         \"boosting_type\": trial.suggest_categorical(\"boosting_type\", [\"Ordered\", \"Plain\"]),\n#         \"bootstrap_type\": trial.suggest_categorical(\n#             \"bootstrap_type\", [\"Bayesian\", \"Bernoulli\", \"MVS\"]\n#         ),\n#     }\n\n#     if param[\"bootstrap_type\"] == \"Bayesian\":\n#         param[\"bagging_temperature\"] = trial.suggest_float(\"bagging_temperature\", 0, 10)\n#     elif param[\"bootstrap_type\"] == \"Bernoulli\":\n#         param[\"subsample\"] = trial.suggest_float(\"subsample\", 0.1, 1)\n\n#     gbm = CatBoostClassifier(**param)\n\n#     gbm.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], \n#             verbose=0, early_stopping_rounds=100)\n\n#     y_pred = gbm.predict(X_valid)\n#     accuracy = accuracy_score(y_valid, y_pred)\n#     return accuracy\n# study = optuna.create_study(direction=\"maximize\")\n# study.optimize(objective, n_trials=100)\n# trial = study.best_trial\n# model = CatBoostClassifier(**trial.params, verbose=False)\n# model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T15:57:10.272207Z","iopub.execute_input":"2023-11-09T15:57:10.272566Z","iopub.status.idle":"2023-11-09T15:58:47.263408Z","shell.execute_reply.started":"2023-11-09T15:57:10.272536Z","shell.execute_reply":"2023-11-09T15:58:47.262093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import log_loss\n# print(log_loss(y_valid, model.predict_proba(X_valid) ))\n","metadata":{"execution":{"iopub.status.busy":"2023-11-06T06:44:29.153343Z","iopub.execute_input":"2023-11-06T06:44:29.154582Z","iopub.status.idle":"2023-11-06T06:44:29.159904Z","shell.execute_reply.started":"2023-11-06T06:44:29.154537Z","shell.execute_reply":"2023-11-06T06:44:29.158566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = model.predict_proba(X_test)\nsubmit = pd.DataFrame({\n    \"id\": df2_ids,\n    \"home\": result[:, 0],\n    \"draw\": result[:, 1],\n    \"away\": result[:, 2]\n})\nsubmit.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T16:01:14.566234Z","iopub.execute_input":"2023-11-09T16:01:14.566663Z","iopub.status.idle":"2023-11-09T16:01:15.837697Z","shell.execute_reply.started":"2023-11-09T16:01:14.566629Z","shell.execute_reply":"2023-11-09T16:01:15.836688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importances = model.get_feature_importance(type='PredictionValuesChange')\nfeature_importances = pd.Series(importances, index=X_train.columns).sort_values()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T16:04:41.348307Z","iopub.execute_input":"2023-11-09T16:04:41.348770Z","iopub.status.idle":"2023-11-09T16:04:41.363146Z","shell.execute_reply.started":"2023-11-09T16:04:41.348719Z","shell.execute_reply":"2023-11-09T16:04:41.361327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importances = feature_importances[feature_importances > 0]","metadata":{"execution":{"iopub.status.busy":"2023-11-09T16:11:04.695616Z","iopub.execute_input":"2023-11-09T16:11:04.696053Z","iopub.status.idle":"2023-11-09T16:11:04.702363Z","shell.execute_reply.started":"2023-11-09T16:11:04.696019Z","shell.execute_reply":"2023-11-09T16:11:04.701198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Most important features\nfeature_importances[-10:]","metadata":{"execution":{"iopub.status.busy":"2023-11-09T16:11:42.992405Z","iopub.execute_input":"2023-11-09T16:11:42.992912Z","iopub.status.idle":"2023-11-09T16:11:43.003877Z","shell.execute_reply.started":"2023-11-09T16:11:42.992879Z","shell.execute_reply":"2023-11-09T16:11:43.002002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(30, 15))\nplt.barh(feature_importances[-10:].index, feature_importances[-10:].values)\nplt.title('CatBoost Feature Importance')\nplt.xlabel('Importance')\nplt.ylabel('Features')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T16:12:21.845429Z","iopub.execute_input":"2023-11-09T16:12:21.845871Z","iopub.status.idle":"2023-11-09T16:12:22.405892Z","shell.execute_reply.started":"2023-11-09T16:12:21.845836Z","shell.execute_reply":"2023-11-09T16:12:22.404168Z"},"trusted":true},"execution_count":null,"outputs":[]}]}