{"cells":[{"metadata":{},"cell_type":"markdown","source":"## 概要\n  * 作成日 : 2020/3/10（火）\n  * 作成者 : 藤根\n  \n## アプローチ内容\n  * チームに関する以下データを整形し、学習に利用する。\n    1. 所属カンファレンス名\n    1. コーチ名\n    1. NCAA出場年数\n    1. シード\n    1. 各シーズンにおける累計得点\n  * 以下の手法で学習・予測する。\n    1. LogisticRegression、RandomForest、SVCなどの複数アルゴリズムを適用したモデルを作る\n    1. グリッドサーチで、各モデルの最適なパラメータを探索する\n    1. 各モデルをアンサンブルし、複数の予測結果から最終的な予測結果（もしくは予測確率）を取得する\n    1. データを訓練／テスト用に分けず、全データで訓練し、全データで予測する（当然、学習済みのデータに対して予測するため、スコアは高くなるはず）\n  \n## 結果\n  * スコアは0.445 -> 0.30018(predict_proba-RandomForestClassifier.csv)\n  * randomforestが今回のデータに適しているというよりも、過学習によってスコアが伸びたようにしか見えない。\n  * 一方で疑問もある。このコード上はrandomforestのaccuracyが1.0なのに、提出時のスコアが0にならないのはなぜだろうか？\n    * 予測すべきチームの組合せが漏れている？\n  \n## 次のアプローチ\n  * ここから更にスコアを上げることは、過学習をさらに深めることと同義である。\n  * よって、Stage-1のアプローチはこの辺で止めた方が良いかもしれない。\n  * やるとしたら、ErrorやWarningを取り除くくらいか。"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import cross_validate, GridSearchCV\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.ensemble import RandomForestClassifier, VotingClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FILEDIR = Path('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# 提出用ファイルを取得\nsub = pd.read_csv(FILEDIR / 'MSampleSubmissionStage1_2020.csv', usecols=['ID'])\nid_splited = sub['ID'].str.split('_', expand=True).astype(int).rename(columns={0: 'Season', 1: 'Team1', 2: 'Team2'})\nsub = pd.concat([sub, id_splited], axis=1).set_index(['Season', 'Team1', 'Team2']).sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# シーズン毎の出場チームを抽出\ntourney_teams = {}\ntourney_teams_all = set()\nfor season in sub.index.get_level_values('Season').drop_duplicates():\n    tourney_teams[season] = set()\n    tourney_teams[season].update(sub.loc[season].index.get_level_values('Team1'))\n    tourney_teams[season].update(sub.loc[season].index.get_level_values('Team2'))\n    tourney_teams_all.update(tourney_teams[season])\n{k: len(v) for k, v in tourney_teams.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 所属カンファレンス情報を取得\nconferences = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeamConferences.csv')\nconferences = pd.concat(\n    [conferences.query('Season == @season and TeamID in @teams') for season, teams in tourney_teams.items()])\nconferences = conferences.set_index(['Season', 'TeamID']).sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# コーチ名を取得\ncoaches = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeamCoaches.csv')\ncoaches = pd.concat(\n    [coaches.query('Season == @season and TeamID in @team') for season, team in tourney_teams.items()])\ncoaches = coaches[coaches['LastDayNum'] == 154].set_index(['Season', 'TeamID']).sort_index()[['CoachName']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NCAAの初回出場年を取得し、初回出場年から現在までの年数を計算\nteams = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeams.csv', usecols=['TeamID', 'FirstD1Season'])\nteams['FirstD1Season'] = 2020 - teams['FirstD1Season']\nteams = pd.concat(\n    [teams.query('TeamID in @team').assign(Season=season) for season, team in tourney_teams.items()])\nteams = teams.set_index(['Season', 'TeamID']).sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 各シーズンのシードを取得\nseeds = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MNCAATourneySeeds.csv')\nseeds = pd.concat(\n    [seeds.query('Season == @season and TeamID in @teams') for season, teams in tourney_teams.items()])\nseeds = seeds.set_index(['Season', 'TeamID']).sort_index()\nseeds['Region'] = seeds['Seed'].str[0]\nseeds['Number'] = seeds['Seed'].str[1:3].astype(int)\ndel seeds['Seed']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# レギュラーシーズンの累計得点と累計失点を取得\nregular = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MRegularSeasonDetailedResults.csv')\nregular = regular.drop(columns=['DayNum', 'LTeamID'])\nregular = pd.concat(\n    [regular.query('Season == @season and WTeamID in @teams') for season, teams in tourney_teams.items()])\nregular = regular.groupby(['Season', 'WTeamID']).sum()\nregular = regular.rename_axis(index=['Season', 'TeamID'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 上記取得データをindexで結合\nctcsr = pd.concat([coaches, teams, conferences, seeds, regular], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NCAAMトーナメントの勝敗結果を取得\nresult = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MNCAATourneyCompactResults.csv')\nresult = result[result['Season'] >= 2015].set_index(['Season', 'WTeamID', 'LTeamID'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 各種データと勝敗結果を結合\nmerged_teams = pd.concat(\n    [ctcsr.loc[[(season, wteam), (season, lteam)], :] for season, wteam, lteam, in result.index])\n\nteam1 = merged_teams.iloc[::2, :].reset_index('TeamID') # teams winned\nteam2 = merged_teams.iloc[1::2, :].reset_index('TeamID') # teams losed\n\nmerged_teams = pd.concat([\n    pd.concat([team1.add_suffix('1'), team2.add_suffix('2')], axis=1).assign(Res=1),\n    pd.concat([team2.add_suffix('1'), team1.add_suffix('2')], axis=1).assign(Res=0),\n]).reset_index().set_index(['Season', 'TeamID1', 'TeamID2']).sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 結合データから、説明変数Xを抽出\n# Xを数値化、正規化\nx_columns = merged_teams.columns[merged_teams.columns != 'Res']\nX = merged_teams[x_columns]\n\nfor column in X.select_dtypes(include='number'):\n    X[column] = MinMaxScaler().fit_transform(X[column].to_numpy().reshape(-1,1))  # FIXME: SettingWithCopyWarning occurred\nX = pd.get_dummies(X, columns=x_columns[X.dtypes == 'object'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 目的変数yを設定\ny = merged_teams['Res']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 学習\nclfs = {}\n\n# SVC\nclfs['SVC'] = {\n    'instance': SVC(probability=True),\n    'params': [\n        {'kernel': ['linear'], 'C': [0.01, 0.05, 0.1, 0.5, 1]},\n        {'kernel': ['rbf'], 'C': [1, 10, 50, 100, 250], 'gamma': [0.1, 0.2, 0.3]}\n    ]    \n}\n\n# RandomForest\nclfs['RandomForestClassifier'] = {\n    'instance': RandomForestClassifier(n_jobs=-1),\n    'params': {        \n        'n_estimators': [25, 50, 100],\n        'criterion': ['gini', 'entropy'],\n        'max_depth': [10, 25, 50, None]\n    }\n}\n\n# LogisticRegression\nclfs['LogisticRegression'] = {\n    'instance': LogisticRegression(max_iter=500, n_jobs=-1),\n    'params': [\n            {'penalty': ['l2'], 'C': [0.1, 0.5, 1, 5, 10]},\n            {'penalty': ['l1'], 'solver': ['liblinear', 'saga'], 'C': [0.1, 0.5, 1, 5, 10]},\n            {'penalty': ['elasticnet'], 'C': [0.1, 0.5, 1, 5, 10], 'l1_ratio': [0.1, 0.3, 0.5, 0.7, 0.9]}\n        ]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for clf_name, clf in clfs.items():\n    print('<{}>'.format(clf_name))\n    print('  training ...'.format(clf_name))\n    \n    gs = GridSearchCV(clf['instance'], param_grid=clf['params'], cv=5, n_jobs=-1)\n    gs.fit(X, y)\n    clfs[clf_name]['best_estimator'] = gs.best_estimator_\n    \n    print('  best_score: {:.3f}'.format(gs.best_score_))\n    print('  best_params: {}'.format(gs.best_params_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 各モデルの予測結果から多数決を採用するソフト分類器を作成\nvote = VotingClassifier(\n    estimators=[(clf_name, clf['best_estimator']) for clf_name, clf in clfs.items()], \n    voting='soft',\n    n_jobs=-1\n)\nvote.fit(X, y)\nvote.estimators_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 各モデルとソフト分類器で、予測結果を比較する\n# randomforestだけ精度が異常に高いのは、多分過学習してる。。。\nfor clf_name, clf in clfs.items():\n    score = accuracy_score(y, clf['best_estimator'].predict(X))\n    print(clf_name, score)\nprint('Vote', accuracy_score(y, vote.predict(X)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 各モデル個別の予測確率と、全モデルを集約した分類器の予測確率を可視化\n# 今回の分類器はソフト投票（各モデルの予測結果の確率の平均をとる）のため、個別のモデルよりもなだらかな分布になっている。\npredict_proba = pd.DataFrame(\n    {clf_name: clf['best_estimator'].predict_proba(X)[:, 1] for clf_name, clf in clfs.items()},\n    index=X.index)\npredict_proba['Vote'] = vote.predict_proba(X)[:, 1]\n_ = predict_proba.plot(kind='hist', bins=50, grid=True, alpha=0.5, figsize=(16,8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 各モデルの予測確率を予測結果ファイルとして出力\ncolumns = predict_proba.columns\nfor column in columns:\n    sub[column] = 0.5\n\nmask = [idx for idx in sub.index if idx in X.index]\nsub.loc[mask, columns] = predict_proba.loc[mask, columns]\n\nfor column in columns:\n    sub[['ID', column]].rename(columns={column: 'pred'}).to_csv('predict_proba-{}.csv'.format(column), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 各モデルの勝敗（0か1）を予測結果ファイルとして出力\npredict = pd.DataFrame(\n    {clf_name: clf['best_estimator'].predict(X) for clf_name, clf in clfs.items()},\n    index=X.index)\npredict['Vote'] = vote.predict(X)\n\ncolumns = predict.columns\nfor column in columns:\n    sub[column] = 0.5\n    \nmask = [idx for idx in sub.index if idx in X.index]\nsub.loc[mask, columns] = predict.loc[mask, columns]\n\nfor column in columns:\n    sub[['ID', column]].rename(columns={column: 'pred'}).to_csv('predict-{}.csv'.format(column), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\ntarget_name = 'predict_proba-RandomForestClassifier.csv'\nnew_name = 'final-submission.csv'\nshutil.copy(target_name, new_name)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}