{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"},"papermill":{"default_parameters":{},"duration":18.385936,"end_time":"2022-05-28T00:10:02.699625","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2022-05-28T00:09:44.313689","version":"2.3.4"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":41875,"databundleVersionId":5521661,"sourceType":"competition"},{"sourceId":7251926,"sourceType":"datasetVersion","datasetId":3648744}],"dockerImageVersionId":30191,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Load data files\n---","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.012678,"end_time":"2022-05-28T00:09:54.541612","exception":false,"start_time":"2022-05-28T00:09:54.528934","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import display_html, HTML\n\nlbs = []\nlbs.append(pd.read_csv(\"/kaggle/input/cafa-5-sub-best/cafa-5-LP.csv\"))\nlbs.append(pd.read_csv(\"/kaggle/input/cafa-5-sub-best/cafa-5-LB_1stupdate.csv\"))\nlbs.append(pd.read_csv(\"/kaggle/input/cafa-5-sub-best/cafa-5-Final.csv\"))\n\ndef print_header(text, size=1):\n    display_html(HTML(\"<h{}>{}</h{}>\".format(size, text, size)))","metadata":{"papermill":{"duration":0.069692,"end_time":"2022-05-28T00:09:54.623249","exception":false,"start_time":"2022-05-28T00:09:54.553557","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-21T05:06:47.238053Z","iopub.execute_input":"2023-12-21T05:06:47.238344Z","iopub.status.idle":"2023-12-21T05:06:47.277592Z","shell.execute_reply.started":"2023-12-21T05:06:47.238313Z","shell.execute_reply":"2023-12-21T05:06:47.276714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Build table where each row contains old and new positions and scores\n---","metadata":{"papermill":{"duration":0.01173,"end_time":"2022-05-28T00:09:54.646718","exception":false,"start_time":"2022-05-28T00:09:54.634988","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Concatenate lbs and remove duplicates to get TeamIDs and TeamNames\nteams = pd.concat(lbs[::-1]).drop_duplicates(subset=\"TeamId\")[['TeamId', 'TeamName']] # Go through lbs in reverse order to obtain latest team names\n\nteamNames = []\nchangeTable = []\n\nfor index, row in teams.iterrows():\n    \n    teamId = row['TeamId']\n    teamName = row['TeamName']\n\n    pos = []\n    score = []\n    \n    for i in range(len(lbs)):\n        elem = lbs[i].loc[lbs[i]['TeamId'] == teamId]\n        \n        if len(elem) > 0:\n            pos.append(elem.index[0] + 1)\n            score.append(elem['Score'].iloc[0])\n        else:\n            pos.append(np.nan)\n            score.append(np.nan)\n\n    teamNames.append(teamName)\n    changeTable.append([teamId] + pos + score)\n    \nchangeTable = np.array(changeTable)\nteamNames = np.array(teamNames)","metadata":{"papermill":{"duration":3.888304,"end_time":"2022-05-28T00:09:58.547057","exception":false,"start_time":"2022-05-28T00:09:54.658753","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-21T05:06:47.279860Z","iopub.execute_input":"2023-12-21T05:06:47.280179Z","iopub.status.idle":"2023-12-21T05:06:49.969056Z","shell.execute_reply.started":"2023-12-21T05:06:47.280137Z","shell.execute_reply":"2023-12-21T05:06:49.968297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lb_lens = []\n\nfor i, lb in enumerate(lbs):\n    print(\"Entries in the LB after update {}:\".format(i + 1), len(lb))\n    lb_lens.append(len(lb))\n\n# Interestingly, a few submissions only failed in earlier updates but not in the latest ones\nplt.title(\"Failed submissions for each update\")\nplt.bar([\"Update {}\".format(i) for i in range(len(lbs))], len(changeTable) - np.array(lb_lens)) # Plot successful submissions with each update\nplt.show()\nlbs[-1].head()","metadata":{"papermill":{"duration":0.216734,"end_time":"2022-05-28T00:09:58.775951","exception":false,"start_time":"2022-05-28T00:09:58.559217","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-21T05:06:49.970391Z","iopub.execute_input":"2023-12-21T05:06:49.970695Z","iopub.status.idle":"2023-12-21T05:06:50.198500Z","shell.execute_reply.started":"2023-12-21T05:06:49.970663Z","shell.execute_reply":"2023-12-21T05:06:50.197510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total entries:\", len(changeTable))\nremove = np.isnan(np.sum(changeTable[:, 2:].astype(np.float32), axis=1))\nteamNames = teamNames[~remove]\nchangeTable = changeTable[~remove]\nprint(\"After removing failed submissions:\", len(changeTable))\nremove = np.min(changeTable[:, 1 + len(lbs):], axis = 1) == -1\nteamNames = teamNames[~remove]\nchangeTable = changeTable[~remove]\nprint(\"After removing -1 submission:\", len(changeTable))","metadata":{"papermill":{"duration":0.02623,"end_time":"2022-05-28T00:09:58.816033","exception":false,"start_time":"2022-05-28T00:09:58.789803","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-21T05:06:50.199844Z","iopub.execute_input":"2023-12-21T05:06:50.200180Z","iopub.status.idle":"2023-12-21T05:06:50.210598Z","shell.execute_reply.started":"2023-12-21T05:06:50.200143Z","shell.execute_reply":"2023-12-21T05:06:50.209617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, k in [(0, 1), (1, 2)]:\n\n    print_header(\"Change from update {} to update {}:\".format(i + 1, k + 1))\n    \n    order = np.argsort(changeTable[:, 1 + i])\n    \n    sortedTeamNames = teamNames[order]\n    sortedChangeTable = changeTable[order]\n\n    print_header(\"Median overall rank/score change:\", 4)\n    print(np.median(np.abs(changeTable[:, 1 + i] - changeTable[:, 1 + k])), \n          round(np.median(np.abs(changeTable[:, 1 + len(lbs) + k] - changeTable[:, 1 + len(lbs) + i])), 4))\n\n    for n in [1000, 500, 200, 100, 50, 20, 10]:\n        print_header(\"Median top {} rank/score change:\".format(n), 4)\n        print(np.median(np.abs(sortedChangeTable[:n, 1 + i] - sortedChangeTable[:n, 1 + k])), \n              round(np.median(np.abs(sortedChangeTable[:n, 1 + len(lbs) + k] - sortedChangeTable[:n, 1 + len(lbs) + i])), 4))\n    \n    print_header(\"Maximum overall rank improvement:\", 4)\n    argtop = np.argmax(sortedChangeTable[:, 1 + i] - sortedChangeTable[:, 1 + k])\n\n    print(\"Team:\", sortedTeamNames[argtop])\n    print(\"Rank:\", int(sortedChangeTable[argtop, 1 + i]), \"->\", int(sortedChangeTable[argtop, 1 + k]))\n    print(\"Score:\", sortedChangeTable[argtop, 1 + len(lbs) + i], \"->\", sortedChangeTable[argtop, 1 + len(lbs) + k])  \n    \n    for n in [1000, 500, 200, 100, 50, 20, 10]:\n        print_header(\"Maximum top {} rank improvement:\".format(n), 4)\n        argtop = np.argmax(sortedChangeTable[:n, 1 + i] - sortedChangeTable[:n, 1 + k])\n\n        print(\"Team:\", sortedTeamNames[argtop])\n        print(\"Rank:\", int(sortedChangeTable[argtop, 1 + i]), \"->\", int(sortedChangeTable[argtop, 1 + k]))\n        print(\"Score:\", sortedChangeTable[argtop, 1 + len(lbs) + i], \"->\", sortedChangeTable[argtop, 1 + len(lbs) + k])  \n    \n    print_header(\"Shakeup rank change histogram\", 4)\n    plt.hist(sortedChangeTable[:, 1 + i] - sortedChangeTable[:, 1 + k], 100)\n    plt.show()\n    \n    print_header(\"Shakeup score change histogram\", 4)\n    plt.hist(sortedChangeTable[:, 1 + len(lbs) + i] - sortedChangeTable[:, 1 + len(lbs) + k], 100)\n    plt.show()","metadata":{"papermill":{"duration":1.567377,"end_time":"2022-05-28T00:10:00.397324","exception":false,"start_time":"2022-05-28T00:09:58.829947","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-21T05:06:50.213097Z","iopub.execute_input":"2023-12-21T05:06:50.213428Z","iopub.status.idle":"2023-12-21T05:06:51.971022Z","shell.execute_reply.started":"2023-12-21T05:06:50.213385Z","shell.execute_reply":"2023-12-21T05:06:51.970261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot the shakeup\n---","metadata":{"papermill":{"duration":0.047022,"end_time":"2022-05-28T00:10:00.493739","exception":false,"start_time":"2022-05-28T00:10:00.446717","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from plotly.offline import init_notebook_mode, iplot, plot\nimport plotly.graph_objs as go","metadata":{"papermill":{"duration":0.091511,"end_time":"2022-05-28T00:10:00.634167","exception":false,"start_time":"2022-05-28T00:10:00.542656","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-21T05:06:51.972190Z","iopub.execute_input":"2023-12-21T05:06:51.972442Z","iopub.status.idle":"2023-12-21T05:06:52.003262Z","shell.execute_reply.started":"2023-12-21T05:06:51.972412Z","shell.execute_reply":"2023-12-21T05:06:52.002271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, k in [(0, 1), (1, 2)]:\n    \n    print_header(\"Change from update {} to update {}:\".format(i + 1, k + 1))\n    \n    # Scatter old vs new rank\n    trace = go.Scatter(x = changeTable[:, 1 + i],\n                       y = changeTable[:, 1 + k],\n                       mode = \"markers\",\n                       name = \"Rank\",\n                       marker = dict(color = 'rgba(128, 128, 255, 0.8)'),\n                       text = np.array(teamNames))\n\n    layout = dict(title = 'Rank Shakeup',\n                  xaxis= dict(title= 'Old Rank',ticklen= 5,zeroline= False),\n                  yaxis= dict(title= 'New Rank',ticklen= 5,zeroline= False))\n\n    fig = dict(data = [trace], layout = layout)\n    iplot(fig)","metadata":{"papermill":{"duration":1.023021,"end_time":"2022-05-28T00:10:01.704458","exception":false,"start_time":"2022-05-28T00:10:00.681437","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-21T05:06:52.004705Z","iopub.execute_input":"2023-12-21T05:06:52.005635Z","iopub.status.idle":"2023-12-21T05:06:52.859502Z","shell.execute_reply.started":"2023-12-21T05:06:52.005596Z","shell.execute_reply":"2023-12-21T05:06:52.858692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, k in [(0, 1), (1, 2)]:\n    \n    print_header(\"Change from update {} to update {}:\".format(i + 1, k + 1))\n    \n    # Scatter old vs new rank\n    trace = go.Scatter(x = changeTable[:, 1 + len(lbs) + i],\n                       y = changeTable[:, 1 + len(lbs) + k],\n                       mode = \"markers\",\n                       name = \"Score\",\n                       marker = dict(color = 'rgba(128, 128, 255, 0.8)'),\n                       text= np.array(teamNames))\n\n    layout = dict(title = 'Score Shakeup',\n                  xaxis= dict(title= 'Old Score',ticklen= 5,zeroline= False),\n                  yaxis= dict(title= 'New Score',ticklen= 5,zeroline= False))\n\n    fig = dict(data = [trace], layout = layout)\n    iplot(fig)","metadata":{"papermill":{"duration":0.253854,"end_time":"2022-05-28T00:10:02.014977","exception":false,"start_time":"2022-05-28T00:10:01.761123","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-21T05:06:52.861107Z","iopub.execute_input":"2023-12-21T05:06:52.861425Z","iopub.status.idle":"2023-12-21T05:06:53.072194Z","shell.execute_reply.started":"2023-12-21T05:06:52.861384Z","shell.execute_reply":"2023-12-21T05:06:53.071280Z"},"trusted":true},"execution_count":null,"outputs":[]}]}