{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":100890,"databundleVersionId":12488425,"sourceType":"competition"},{"sourceId":50947,"databundleVersionId":5741538,"sourceType":"competition"},{"sourceId":11996390,"sourceType":"datasetVersion","datasetId":9}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on May 29, 2025. By Prata, Marília (mpwolke)","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n#Two lines Required to Plot Plotly\nimport plotly.io as pio\npio.renderers.default = 'iframe'\n\nimport plotly.graph_objects as go\nimport plotly.express as px\n\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=Warning)\n\nimport cv2\nfrom wordcloud import WordCloud, ImageColorGenerator\n\nimport urllib\nimport random\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-30T00:10:55.128528Z","iopub.execute_input":"2025-05-30T00:10:55.128951Z","iopub.status.idle":"2025-05-30T00:10:55.196389Z","shell.execute_reply.started":"2025-05-30T00:10:55.128926Z","shell.execute_reply":"2025-05-30T00:10:55.195001Z"},"_kg_hide-input":true,"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"![](https://slideplayer.com/slide/9981175/32/images/2/kaggle+What+is+Kaggle+A+data+science+competitions+%3A.jpg)Slideplayer","metadata":{}},{"cell_type":"markdown","source":"## Competition Citation\n\n@misc{meta-kaggle-hackathon,\n\n author = {Paul Mooney and Meg Risdal and Maria Cruz and Addison Howard},\n \n title = {Meta Kaggle Hackathon},\n \n year = {2025},\n howpublished = {\\url{https://kaggle.com/competitions/meta-kaggle-hackathon}},\n note = {Kaggle}\n}\n\n\"The Meta Kaggle and Meta Kaggle Code datasets offer a unique perspective for studying these **\"gold standard\" evaluations.** Each competition that Kaggle hosts represents a challenging problem valuable to the larger scientific community. The progress made in these competitions, as visualized by climbing leaderboards, and the vast amounts of metadata surrounding them, offers a unique opportunity to understand how the industry and community have evolved.\"\n\nhttps://www.kaggle.com/competitions/meta-kaggle-hackathon/overview","metadata":{}},{"cell_type":"markdown","source":"#My Gold Standard Contributors to MetaKaggle with their substantial Kaggle Notebooks\n\nJames Trotman (jtrotman)\n\nBwandoWando (bwandowando)\n\nSteubk (steubk)\n\nGabriel Preda (gpreda)\n\nCarl McBride Ellis (carlmcbrideellis)\n\nPaul Mooney (paulthimothymooney)\n\nBen Hamner (benhamner)\n\nAnd Last, but not least: Kaggle Kerneler (kerneler)","metadata":{}},{"cell_type":"markdown","source":"# What about Competition Write-ups?\n\nI added 2023 Kaggle AI Report to enrich this Gold Standard Evaluations.","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv\", delimiter=',', encoding='utf8')\npd.set_option('display.max_columns', None)\ndf.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T00:11:40.838703Z","iopub.execute_input":"2025-05-30T00:11:40.839035Z","iopub.status.idle":"2025-05-30T00:11:41.296585Z","shell.execute_reply.started":"2025-05-30T00:11:40.839012Z","shell.execute_reply":"2025-05-30T00:11:41.295406Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#First Place Solution title on top. Could you predict that? : )","metadata":{}},{"cell_type":"code","source":"df[\"Title of Writeup\"].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T00:14:06.298968Z","iopub.execute_input":"2025-05-30T00:14:06.299312Z","iopub.status.idle":"2025-05-30T00:14:06.311621Z","shell.execute_reply.started":"2025-05-30T00:14:06.299282Z","shell.execute_reply":"2025-05-30T00:14:06.310542Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##Code by Taha07 https://www.kaggle.com/taha07/data-scientists-jobs-analysis-visualization/notebook\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'white',\n color_func=lambda *args, **kwargs: \"black\",\n height =2000,\n width = 2000\n ).generate(str(df[\"Title of Writeup\"]))\nplt.rcParams['figure.figsize'] = (12,12)\nplt.axis(\"off\")\nplt.imshow(wordcloud)\nplt.title(\"Kaggle Competitions Writeups titles\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T00:35:58.240363Z","iopub.execute_input":"2025-05-30T00:35:58.240730Z","iopub.status.idle":"2025-05-30T00:36:01.642333Z","shell.execute_reply.started":"2025-05-30T00:35:58.240702Z","shell.execute_reply":"2025-05-30T00:36:01.641276Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## AI Competitions, the gold standard for empirical rigor in GenAI evaluation\n\nPosition: AI Competitions Provide the Gold Standard for Empirical Rigor in GenAI Evaluation\n\nAuthors: D. Sculley, Will Cukierski, Phil Culliton, Sohier Dane, Maggie Demkin, Ryan Holbrook,\nAddison Howard, Paul Mooney, Walter Reade, Megan Risdal, Nate Keating.\n\n\"In this position paper, the authors observed that empirical evaluation in Generative AI is at a crisis point since traditional ML evaluation and benchmarking strategies are insufficient to meet the needs of evaluating modern GenAI models and systems.\"\n\n\"There are many reasons for this, including the fact that these models typically have nearly unbounded input and output spaces, typically do not have a well defined ground truth target, and typically exhibit strong feedback loops and prediction dependence based on context of previous model\noutputs.\" \n\n**Recommendations for the Field**\n\n\"Move away from static benchmarks and towards evergreen repeatable processes. View the steady stream of AI Competitions as a resource for the field. Adopt and improve on the anti-cheating structures from AI Competitions to improve standard practice for GenAI evaluations.\"\n\n\"The many new static benchmarks are appearing on platforms like **Hugging Face, OpenML, and Kaggle** on a near-daily basis and may serve as the steady stream of novel tasks that the authors described as\nnecessary for the field. While they applaud all efforts to create new benchmarks, they do fundamentally believe that static benchmarks should be considered to have been effectively invalidated once they have been published, and thus it is the time component of AI Competitions that provides unique additional value.\"\n\n\"Without controlled, empirical study the authors shared knowledge into why models **perform well or poorly on certain tasks**. Openly sharing this understanding is critical for unlocking paths to further progress in this rapidly advancing field.\"\n\nhttps://arxiv.org/pdf/2505.00612v1","metadata":{}},{"cell_type":"markdown","source":"#The snippet below by SRK is one of my favorites of Kaggle Competitions\n\nI had only to add meta-kaggle to the files.","metadata":{}},{"cell_type":"code","source":"#By SRK https://www.kaggle.com/code/sudalairajkumar/winning-solutions-of-kaggle-competitions\n\nfrom IPython.core.display import HTML\n\ndata_path = \"../input/\"\n\ncompetitions_df = pd.read_csv(data_path + \"meta-kaggle/Competitions.csv\")\ncomps_to_use = [\"Featured\", \"Research\", \"Recruitment\"]\ncompetitions_df = competitions_df[competitions_df[\"HostSegmentTitle\"].isin(comps_to_use)]\ncompetitions_df[\"EnabledDate\"] = pd.to_datetime(competitions_df[\"EnabledDate\"], format=\"%m/%d/%Y %H:%M:%S\")\ncompetitions_df = competitions_df.sort_values(by=\"EnabledDate\", ascending=False).reset_index(drop=True)\ncompetitions_df.head()\n\nforum_topics_df = pd.read_csv(data_path + \"meta-kaggle/ForumTopics.csv\")\n\ncomp_tags_df = pd.read_csv(data_path + \"meta-kaggle/CompetitionTags.csv\")\ntags_df = pd.read_csv(data_path + \"meta-kaggle/Tags.csv\", usecols=[\"Id\", \"Name\"])\n\ndef get_comp_tags(comp_id):\n temp_df = comp_tags_df[comp_tags_df[\"CompetitionId\"]==comp_id]\n temp_df = pd.merge(temp_df, tags_df, left_on=\"TagId\", right_on=\"Id\")\n tags_str = \"Tags : \"\n for ind, row in temp_df.iterrows():\n tags_str += row[\"Name\"] + \", \"\n return tags_str.strip(\", \")\n\ndef check_solution(topic):\n is_solution = False\n to_exclude = [\"?\", \"submit\", \"why\", \"what\", \"resolution\", \"benchmark\"]\n if \"solution\" in topic.lower():\n is_solution = True\n for exc in to_exclude:\n if exc in topic.lower():\n is_solution = False\n to_include = [\"2nd place code\", '\"dance with ensemble\" sharing']\n for inc in to_include:\n if inc in topic.lower():\n is_solution = True\n return is_solution\n\ndef get_discussion_results(forum_id, n):\n results_df = forum_topics_df[forum_topics_df[\"ForumId\"]==forum_id]\n results_df[\"is_solution\"] = results_df[\"Title\"].apply(lambda x: check_solution(str(x)))\n results_df = results_df[results_df[\"is_solution\"] == 1]\n results_df = results_df.sort_values(by=[\"Score\",\"TotalMessages\"], ascending=False).head(n).reset_index(drop=True)\n return results_df[[\"Title\", \"Id\", \"Score\", \"TotalMessages\", \"TotalReplies\"]]\n\ndef render_html_for_comp(forum_id, comp_id, comp_name, comp_slug, comp_subtitle, n):\n results_df = get_discussion_results(forum_id, n)\n \n if len(results_df) < 1:\n return\n \n comp_tags = get_comp_tags(comp_id)\n \n comp_url = \"https://www.kaggle.com/c/\"+str(comp_slug)\n hs = \"\"\"\n

\"\"\"+comp_name+\"\"\"

\n

\"\"\"+comp_subtitle+\"\"\"

\n \"\"\"\n \n if comp_tags != \"Tags :\":\n hs += \"\"\"\n

\"\"\"+comp_tags+\"\"\"

\n \"\"\"\n \n hs += \"\"\"\n \n \n \n \n \n \n \"\"\"\n \n for i, row in results_df.iterrows():\n url = \"https://www.kaggle.com/c/\"+str(comp_slug)+\"/discussion/\"+str(row[\"Id\"])\n hs += \"\"\"\n \n \n \n \n \"\"\"\n hs += \"
S.NoDiscussion TitleNumber of upvotesTotal Replies
\"\"\"+str(i+1)+\"\"\"\"\"\" +str(row['Title']) + \"\"\"\"\"\"+str(row['Score'])+\"\"\"\"\"\"+str(row['TotalReplies'])+\"\"\"
\"\n display(HTML(hs))\n\nfor ind, comp_row in competitions_df.iterrows():\n render_html_for_comp(comp_row[\"ForumId\"], comp_row[\"Id\"], comp_row[\"Title\"], comp_row[\"Slug\"], comp_row[\"Subtitle\"], 12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-30T00:52:36.094281Z","iopub.execute_input":"2025-05-30T00:52:36.094674Z","iopub.status.idle":"2025-05-30T00:52:42.936813Z","shell.execute_reply.started":"2025-05-30T00:52:36.094645Z","shell.execute_reply":"2025-05-30T00:52:42.935867Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Draft Session 2h:7m","metadata":{}},{"cell_type":"markdown","source":"#Acknowledgements:\n\nSRK https://www.kaggle.com/code/sudalairajkumar/winning-solutions-of-kaggle-competitions","metadata":{}}]}