{"cells":[{"metadata":{},"cell_type":"markdown","source":"**https://www.kaggle.com/takaishikawa/no-ml-modeling**"},{"metadata":{},"cell_type":"markdown","source":"**This notebook predicts the probability of the occurrence of the upset, which means that the low seed rank team beats the high seed rank team, aggregating past game results**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.metrics import log_loss\nfrom tqdm import tqdm\nimport warnings","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"config = {\n    \"mode\": {\n        \"stage\": 1\n    },\n    \"const\": {\n        \"score_diff\": 5,\n        \"this_season\": 2020,\n        \"total_season\": 10,\n        \"seed_num\": 16,\n        \"clip_min\": 0.01,\n        \"clip_max\": 0.99,\n    },\n    \"path\": {\n        \"prefix\": \"/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament\",\n        \"stage1_prefix\": f\"/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1\",\n    },\n    \"plot\": {\n        \"palette\": \"viridis_r\"\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nsns.set()\nwarnings.filterwarnings(\"ignore\")\n\npath_prefix = config[\"path\"][\"prefix\"]\nstage1_prefix = config[\"path\"][\"stage1_prefix\"]\nprint(os.listdir(f\"{path_prefix}\"))\nprint(os.listdir(f\"{stage1_prefix}\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(pd.read_csv(f\"{path_prefix}/MSampleSubmissionStage1_2020.csv\").shape)\ndisplay(pd.read_csv(f\"{path_prefix}/MSampleSubmissionStage1_2020.csv\").head())\ndisplay(pd.read_csv(f\"{path_prefix}/MSampleSubmissionStage1_2020.csv\").tail())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data():\n    df_seed = pd.read_csv(os.path.join(stage1_prefix, \"MNCAATourneySeeds.csv\"))\n    df_result = pd.read_csv(\n        os.path.join(stage1_prefix, \"MNCAATourneyCompactResults.csv\")\n    )\n    return df_seed, df_result\n\n\ndef _seed_to_int(seed):\n    s_int = int(seed[1:3])\n    return s_int\n\n\ndef clean_df(df_seed, df_result):\n    df_seed[\"seed_int\"] = df_seed[\"Seed\"].apply(_seed_to_int)\n    df_seed.drop([\"Seed\"], axis=1, inplace=True)\n    df_result.drop([\"DayNum\", \"WLoc\", \"NumOT\"], axis=1, inplace=True)\n    return df_seed, df_result\n\n\n# Merge seed for each team\ndef merge_seed_result(df_seed, df_result):\n    df_win_seed = df_seed.rename(columns={\"TeamID\": \"WTeamID\", \"seed_int\": \"WSeed\"})\n    df_loss_seed = df_seed.rename(columns={\"TeamID\": \"LTeamID\", \"seed_int\": \"LSeed\"})\n    df_result = df_result.merge(df_win_seed, how=\"left\", on=[\"Season\", \"WTeamID\"])\n    df_result = df_result.merge(df_loss_seed, how=\"left\", on=[\"Season\", \"LTeamID\"])\n    df_result[\"SeedDiff\"] = np.abs(df_result[\"WSeed\"] - df_result[\"LSeed\"])\n    df_result[\"ScoreDiff\"] = np.abs(df_result[\"WScore\"] - df_result[\"LScore\"])\n    return df_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_seed, df_result = load_data()\ndf_seed, df_result = clean_df(df_seed, df_result)\ndf_result = merge_seed_result(df_seed, df_result)\ndf_result[\"upset\"] = [\n    1 if ws > ls else 0 for ws, ls, in zip(df_result[\"WSeed\"], df_result[\"LSeed\"])\n]\n\n# Remove the games that end within 3 points difference, which are likely to be the other results\ndf_result = df_result[df_result[\"ScoreDiff\"] > config[\"const\"][\"score_diff\"]]\ndf_result.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_target(df_result):\n    upset_proba = df_result['upset'].value_counts() / len(df_result) * 100\n    print(f\"upset probability:\\n{upset_proba}\")\n\ncheck_target(df_result)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Upset is more likely to happen in men's tournament than [women's](https://www.kaggle.com/takaishikawa/no-ml-modeling-ncaaw2020)**"},{"metadata":{},"cell_type":"markdown","source":"### Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use only last 10 seasons, since some trends are likely to be changed\nthis_season = 2015\ntotal_season = config[\"const\"][\"total_season\"]\nseed_num = config[\"const\"][\"seed_num\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nThe probability of the occurrence of the upset is likely to be different between\na game 1st seed vs. 6th seed and a game 11th seed vs. 16th seed, \nso I want to include the information\n\"\"\"\ndef aggregation(df_result, plot=True):\n    df_result[\"Seed_combi\"] = [\n        str(ws) + \"_\" + str(ls) if ws < ls else str(ls) + \"_\" + str(ws)\n        for ws, ls in zip(df_result[\"WSeed\"], df_result[\"LSeed\"])\n    ]\n\n    df_result_aggs = pd.DataFrame()\n    df_result_filter_aggs = pd.DataFrame()\n    df_result_season = df_result[\n        (df_result[\"Season\"] >= (this_season - total_season))\n        & (df_result[\"Season\"] < (this_season - 1))\n    ]\n    for s_num in range(seed_num):\n        df_result_agg = (\n            df_result_season[df_result_season[\"SeedDiff\"] == s_num]\n            .groupby(\"SeedDiff\")\n            .agg({\"upset\": [\"mean\", \"count\"]})\n        )\n        df_result_agg.columns = [\n            f\"{col[0]}_{col[1]}_all\" for col in df_result_agg.columns\n        ]\n        df_result_filter_agg = (\n            df_result_season[df_result_season[\"SeedDiff\"] == s_num]\n            .groupby(\"Seed_combi\")\n            .agg({\"upset\": [\"mean\", \"count\"]})\n        )\n        df_result_filter_agg.columns = [\n            f\"{col[0]}_{col[1]}\" for col in df_result_filter_agg.columns\n        ]\n        if s_num == 0:\n            df_result_agg[\"upset_mean_all\"] = 0.5\n            df_result_filter_agg[\"upset_mean\"] = 0.5\n        df_result_aggs = pd.concat([df_result_aggs, df_result_agg])\n        df_result_filter_aggs = pd.concat([df_result_filter_aggs, df_result_filter_agg])\n\n    if plot:\n        sns.barplot(df_result_aggs.index, df_result_aggs.upset_mean_all, palette=config[\"plot\"][\"palette\"])\n        plt.title(\"probability of upset based on past result aggretation\")\n        plt.tight_layout()\n        plt.show()\n\n    return df_result_aggs, df_result_filter_aggs\n\n\n# Merge upset probability\ndef merge(df_result, df_result_aggs, df_result_filter_aggs):\n    df_result = df_result.join(df_result_aggs, how=\"left\", on=\"SeedDiff\").join(\n        df_result_filter_aggs, how=\"left\", on=\"Seed_combi\"\n    )\n    df_result[\"upset_prob\"] = [\n        m if c > 20 else a\n        for a, m, c in zip(\n            df_result[\"upset_mean_all\"],\n            df_result[\"upset_mean\"],\n            df_result[\"upset_count\"],\n        )\n    ]\n    valid = df_result[(df_result[\"Season\"] == (this_season - 1))]\n    return valid\n\n\n# heuristic smoothing\ndef smoothing(df_result_aggs, plot=True):\n    for i in range(config[\"const\"][\"seed_num\"]):\n        if i == 0:\n            df_result_aggs.loc[i, \"upset_mean_all\"] = 0.5\n        else:\n            try:\n                df_result_aggs.loc[i, \"upset_mean_all\"]\n                if df_result_aggs.loc[i, \"upset_mean_all\"] == 0:\n                    raise Exception\n                elif df_result_aggs.loc[i, \"upset_mean_all\"] > 0.5:\n                    df_result_aggs.loc[i, \"upset_mean_all\"] = 0.5\n            except Exception:\n                df_result_aggs.loc[i, \"upset_mean_all\"] = (\n                    df_result_aggs.loc[(i - 1), \"upset_mean_all\"] / 4\n                    + df_result_aggs.loc[(i - 2), \"upset_mean_all\"] / 4\n                )\n\n    if plot:\n        sns.barplot(df_result_aggs.index, df_result_aggs.upset_mean_all, palette=config[\"plot\"][\"palette\"])\n        plt.title(\"probability of upset based on past result aggretation\")\n        plt.tight_layout()\n        plt.show()\n\n    return df_result_aggs\n\n\ndef merge_smooting(df_result, df_result_aggs_smooth, df_result_filter_aggs):\n    df_result = df_result.join(df_result_aggs_smooth, how=\"left\", on=\"SeedDiff\").join(\n        df_result_filter_aggs, how=\"left\", on=\"Seed_combi\"\n    )\n    df_result[\"upset_prob\"] = [\n        m if c > 20 else a\n        for a, m, c in zip(\n            df_result[\"upset_mean_all\"],\n            df_result[\"upset_mean\"],\n            df_result[\"upset_count\"],\n        )\n    ]\n\n    valid = df_result[(df_result[\"Season\"] == (this_season - 1))]\n    return valid\n\n\ndef clipping(array, a_min=config[\"const\"][\"clip_min\"], a_max=config[\"const\"][\"clip_max\"]):\n    return np.clip(array, a_min, a_max)\n\n\ndef scoring(valid, clip=False):\n    if clip:\n        return log_loss(valid[\"upset\"], clipping(valid[\"upset_prob\"]))\n    else:\n        return log_loss(valid[\"upset\"], valid[\"upset_prob\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_result_aggs, df_result_filter_aggs = aggregation(df_result, plot=True)\n# valid = merge(df_result, df_result_aggs, df_result_filter_aggs)\n# print(scoring(valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_result_aggs, df_result_filter_aggs = aggregation(df_result, plot=True)\ndf_result_aggs_smooth = smoothing(df_result_aggs, plot=True)\nvalid = merge_smooting(df_result, df_result_aggs_smooth, df_result_filter_aggs)\nprint(scoring(valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_result_aggs, df_result_filter_aggs = aggregation(df_result, plot=True)\ndf_result_aggs_smooth = smoothing(df_result_aggs, plot=True)\nvalid = merge_smooting(df_result, df_result_aggs_smooth, df_result_filter_aggs)\nprint(scoring(valid, clip=True))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_seed_test = df_seed[df_seed[\"Season\"]==this_season]\ndf_result_aggs, df_result_filter_aggs = aggregation(df_result, plot=True)\ndf_result_aggs_smooth = smoothing(df_result_aggs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_test(df_seed_this_season, df_result_aggs, df_result_filter_aggs):\n    test = pd.read_csv(os.path.join(path_prefix, \"MSampleSubmissionStage1_2020.csv\"))\n    test = pd.DataFrame(\n        np.array([ID.split(\"_\") for ID in test[\"ID\"]]),\n        columns=[\"Season\", \"TeamA\", \"TeamB\"],\n        dtype=int,\n    )\n\n    test = test.merge(\n        df_seed_this_season,\n        how=\"left\",\n        left_on=[\"Season\", \"TeamA\"],\n        right_on=[\"Season\", \"TeamID\"],\n    )\n    test = test.rename(columns={\"seed_int\": \"TeamA_seed\"}).drop(\"TeamID\", axis=1)\n\n    test = test.merge(\n        df_seed_this_season,\n        how=\"left\",\n        left_on=[\"Season\", \"TeamB\"],\n        right_on=[\"Season\", \"TeamID\"],\n    )\n    test = test.rename(columns={\"seed_int\": \"TeamB_seed\"}).drop(\"TeamID\", axis=1)\n\n    test[\"SeedDiff\"] = np.abs(test.TeamA_seed - test.TeamB_seed)\n    test[\"Seed_combi\"] = [\n        str(a) + \"_\" + str(b) if a < b else str(b) + \"_\" + str(a)\n        for a, b in zip(test[\"TeamA_seed\"], test[\"TeamB_seed\"])\n    ]\n\n    test = (\n        test.join(df_result_aggs, how=\"left\", on=\"SeedDiff\")\n        .join(df_result_filter_aggs, how=\"left\", on=\"Seed_combi\")\n        .fillna(-1)\n    )\n    test[\"upset_prob\"] = [\n        m if c > 20 else a\n        for a, m, c in zip(\n            test[\"upset_mean_all\"], test[\"upset_mean\"], test[\"upset_count\"]\n        )\n    ]\n\n    # convert upset_prob to win_prob\n    test[\"win_prob\"] = [\n        (1 - upset_prob) if teamA < teamB else upset_prob if teamA > teamB else 0.5\n        for teamA, teamB, upset_prob in zip(\n            test[\"TeamA_seed\"], test[\"TeamB_seed\"], test[\"upset_prob\"]\n        )\n    ]\n\n    return test\n\n\ndef make_submit(test, clip):\n    if clip:\n        sub = clipping(test[\"win_prob\"].values)\n    else:\n        sub = test[\"win_prob\"].values\n    submit = pd.read_csv(os.path.join(path_prefix, \"MSampleSubmissionStage1_2020.csv\"))\n    submit[\"Pred\"] = sub\n    start_index = int((len(submit) / 5) * (this_season - 2015))\n    end_index = int((len(submit) / 5) * (this_season - 2015 + 1))\n    submit_this_season = submit.iloc[start_index:end_index, :]\n    return submit_this_season","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = load_test(df_seed_test, df_result_aggs_smooth, df_result_filter_aggs)\nmake_submit(test, clip=False)\nmake_submit(test, clip=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"seasons = [2015, 2016, 2017, 2018, 2019]\nscores = []\nscores_clip = []\nsubmits = []\nsubmits_clip = []\nfor this_season in tqdm(seasons):\n    df_seed, df_result = load_data()\n    df_seed, df_result = clean_df(df_seed, df_result)\n    df_result = merge_seed_result(df_seed, df_result)\n    df_result['upset'] = [1 if ws > ls else 0 for ws, ls, in zip(df_result[\"WSeed\"], df_result[\"LSeed\"])]\n    df_result = df_result[df_result['ScoreDiff'] > config[\"const\"][\"score_diff\"]]\n\n    df_result_aggs, df_result_filter_aggs = aggregation(df_result, plot=False)\n    df_result_aggs_smooth = smoothing(df_result_aggs, plot=False)\n    valid = merge_smooting(df_result, df_result_aggs_smooth, df_result_filter_aggs)\n    score = scoring(valid, clip=False)\n    scores.append(score)\n    print(f\"{this_season}: {score} without clipping\")\n    score_clip = scoring(valid, clip=True)\n    scores_clip.append(score_clip)\n    print(f\"{this_season}: {score_clip} with clipping\")\n    \n    df_seed_test = df_seed[df_seed[\"Season\"]==this_season]\n    df_result_aggs, df_result_filter_aggs = aggregation(df_result, plot=False)\n    df_result_aggs_smooth = smoothing(df_result_aggs, plot=False)\n    test = load_test(df_seed_test, df_result_aggs_smooth, df_result_filter_aggs)\n    submit_this_season = make_submit(test, clip=False)\n    submits.append(submit_this_season)\n    submit_this_season = make_submit(test, clip=True)\n    submits_clip.append(submit_this_season)\n\nprint(f\"cv all without clipping: {round(np.mean(scores), 6)}\")\nprint(f\"cv all with clipping: {round(np.mean(scores_clip), 6)}\")\n\nsubmit = pd.concat(submits, axis=0)\nfilename = \"submission_agg_all_manually_nocliped.csv\"\nsubmit.to_csv(filename, index=False)\n\nsubmit_clip = pd.concat(submits_clip, axis=0)\nfilename = \"submission_agg_all_manually_cliped.csv\"\nsubmit_clip.to_csv(filename, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}