{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Table of Contents\n* [Arxiv Metadata (JSON file)](#arxiv)\n* [Kaggle Writeups](#kaggle)","metadata":{}},{"cell_type":"code","source":"# packages\n\nimport numpy as np\nimport pandas as pd\nimport time\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport json\nimport gc\n\nfrom wordcloud import WordCloud, STOPWORDS, ImageColorGenerator","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-12T12:19:51.372755Z","iopub.execute_input":"2023-05-12T12:19:51.373139Z","iopub.status.idle":"2023-05-12T12:19:51.378297Z","shell.execute_reply.started":"2023-05-12T12:19:51.373109Z","shell.execute_reply":"2023-05-12T12:19:51.377229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# file overview\n!ls -l '../input/2023-kaggle-ai-report'","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:19:53.356256Z","iopub.execute_input":"2023-05-12T12:19:53.356613Z","iopub.status.idle":"2023-05-12T12:19:55.139769Z","shell.execute_reply.started":"2023-05-12T12:19:53.356585Z","shell.execute_reply":"2023-05-12T12:19:55.138391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='arxiv'></a>\n# Arxiv Metadata (JSON file)","metadata":{}},{"cell_type":"code","source":"# import JSON data\nfile_name = '../input/2023-kaggle-ai-report/arxiv_metadata_20230510.json'\nt1 = time.time()\ndict_arxiv = []\nfor line in open(file_name, 'r'):\n    dict_arxiv.append(json.loads(line))\nt2 = time.time()\nprint('Elapsed time:', np.round(t2-t1,2))","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:19:55.265096Z","iopub.execute_input":"2023-05-12T12:19:55.265559Z","iopub.status.idle":"2023-05-12T12:21:43.554578Z","shell.execute_reply.started":"2023-05-12T12:19:55.265515Z","shell.execute_reply":"2023-05-12T12:21:43.553416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show entry\ndict_arxiv[0]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:21:43.559339Z","iopub.execute_input":"2023-05-12T12:21:43.559828Z","iopub.status.idle":"2023-05-12T12:21:43.569696Z","shell.execute_reply.started":"2023-05-12T12:21:43.559770Z","shell.execute_reply":"2023-05-12T12:21:43.568601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert to data frame\ndf_arxiv = pd.DataFrame.from_dict(dict_arxiv)\n# and clean up\ndel dict_arxiv\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:21:43.571251Z","iopub.execute_input":"2023-05-12T12:21:43.571572Z","iopub.status.idle":"2023-05-12T12:21:54.930812Z","shell.execute_reply.started":"2023-05-12T12:21:43.571544Z","shell.execute_reply":"2023-05-12T12:21:54.929577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_arxiv.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:21:54.935251Z","iopub.execute_input":"2023-05-12T12:21:54.935610Z","iopub.status.idle":"2023-05-12T12:21:55.258647Z","shell.execute_reply.started":"2023-05-12T12:21:54.935582Z","shell.execute_reply":"2023-05-12T12:21:55.257529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure\ndf_arxiv.info(show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:21:55.259907Z","iopub.execute_input":"2023-05-12T12:21:55.260194Z","iopub.status.idle":"2023-05-12T12:22:02.417023Z","shell.execute_reply.started":"2023-05-12T12:21:55.260169Z","shell.execute_reply":"2023-05-12T12:22:02.416222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# categories - show top 50 only\ndf_arxiv.categories.value_counts()[0:50]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:22:02.418335Z","iopub.execute_input":"2023-05-12T12:22:02.419313Z","iopub.status.idle":"2023-05-12T12:22:03.125990Z","shell.execute_reply.started":"2023-05-12T12:22:02.419279Z","shell.execute_reply":"2023-05-12T12:22:03.124878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make available for download as CSV\nt1 = time.time()\ndf_arxiv.to_csv('df_arxiv.csv', index=False)\nt2 = time.time()\nprint('Elapsed time:', np.round(t2-t1,2))","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:22:03.127612Z","iopub.execute_input":"2023-05-12T12:22:03.128089Z","iopub.status.idle":"2023-05-12T12:23:48.694742Z","shell.execute_reply.started":"2023-05-12T12:22:03.128057Z","shell.execute_reply":"2023-05-12T12:23:48.693574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Select a subset:","metadata":{}},{"cell_type":"code","source":"# filter e.g. for Computer Science / AI\ndf_select = df_arxiv[df_arxiv.categories=='cs.AI'].reset_index()\ndf_select","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:24:17.453472Z","iopub.execute_input":"2023-05-12T12:24:17.454032Z","iopub.status.idle":"2023-05-12T12:24:18.043496Z","shell.execute_reply.started":"2023-05-12T12:24:17.453994Z","shell.execute_reply":"2023-05-12T12:24:18.041389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show wordcloud for this selection\ntext = \" \".join(xx for xx in df_select.abstract)\ntext\n\nstopwords_cloud = set(STOPWORDS)\n\nwordcloud = WordCloud(stopwords=stopwords_cloud, max_font_size=50, max_words=250,\n                      width = 600, height = 400,\n                      background_color='black').generate(text)\nplt.figure(figsize=(12,8))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:24:24.711057Z","iopub.execute_input":"2023-05-12T12:24:24.711380Z","iopub.status.idle":"2023-05-12T12:24:31.322275Z","shell.execute_reply.started":"2023-05-12T12:24:24.711352Z","shell.execute_reply":"2023-05-12T12:24:31.321112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='kaggle'></a>\n# Kaggle Writeups","metadata":{}},{"cell_type":"code","source":"# load data\ndf_writeups = pd.read_csv('../input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv')","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:25:36.979288Z","iopub.execute_input":"2023-05-12T12:25:36.979662Z","iopub.status.idle":"2023-05-12T12:25:37.357548Z","shell.execute_reply.started":"2023-05-12T12:25:36.979634Z","shell.execute_reply":"2023-05-12T12:25:37.356359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf_writeups.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:25:37.764686Z","iopub.execute_input":"2023-05-12T12:25:37.765284Z","iopub.status.idle":"2023-05-12T12:25:37.777279Z","shell.execute_reply.started":"2023-05-12T12:25:37.765252Z","shell.execute_reply":"2023-05-12T12:25:37.776162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure\ndf_writeups.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:25:39.180959Z","iopub.execute_input":"2023-05-12T12:25:39.181316Z","iopub.status.idle":"2023-05-12T12:25:39.197995Z","shell.execute_reply.started":"2023-05-12T12:25:39.181287Z","shell.execute_reply":"2023-05-12T12:25:39.196643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show top 50 competitions by count\ndf_writeups['Title of Competition'].value_counts()[0:50]","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:25:40.502022Z","iopub.execute_input":"2023-05-12T12:25:40.502395Z","iopub.status.idle":"2023-05-12T12:25:40.510973Z","shell.execute_reply.started":"2023-05-12T12:25:40.502363Z","shell.execute_reply":"2023-05-12T12:25:40.510094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Select a specific competition:","metadata":{}},{"cell_type":"code","source":"df_select = df_writeups[df_writeups['Title of Competition']=='M5 Forecasting - Accuracy'].reset_index()\ndf_select","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:25:45.212393Z","iopub.execute_input":"2023-05-12T12:25:45.212767Z","iopub.status.idle":"2023-05-12T12:25:45.235701Z","shell.execute_reply.started":"2023-05-12T12:25:45.212737Z","shell.execute_reply":"2023-05-12T12:25:45.234587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show wordcloud for this selection\ntext = \" \".join(xx for xx in df_select['Writeup'])\ntext\n\nstopwords_cloud = set(STOPWORDS)\nstopwords_cloud.add('li')\nstopwords_cloud.add('ul')\nstopwords_cloud.add('ol')\nstopwords_cloud.add('em')\nstopwords_cloud.add('p')\nstopwords_cloud.add('b')\nstopwords_cloud.add('br')\nstopwords_cloud.add('h2')\nstopwords_cloud.add('h3')\nstopwords_cloud.add('href')\nstopwords_cloud.add('https')\nstopwords_cloud.add('o')\nstopwords_cloud.add('img')\nstopwords_cloud.add('src')\nstopwords_cloud.add('alt')\n\nwordcloud = WordCloud(stopwords=stopwords_cloud, max_font_size=50, max_words=250,\n                      width = 600, height = 400,\n                      background_color='black').generate(text)\nplt.figure(figsize=(12,8))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-12T12:37:22.983634Z","iopub.execute_input":"2023-05-12T12:37:22.984347Z","iopub.status.idle":"2023-05-12T12:37:23.983143Z","shell.execute_reply.started":"2023-05-12T12:37:22.984302Z","shell.execute_reply":"2023-05-12T12:37:23.981997Z"},"trusted":true},"execution_count":null,"outputs":[]}]}