{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Detailed** analysis of ARXIV articles on calculus and language and articles.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nimport time\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\nfrom tqdm.notebook import tqdm\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:38:39.079680Z","iopub.execute_input":"2023-07-08T13:38:39.080013Z","iopub.status.idle":"2023-07-08T13:38:39.224520Z","shell.execute_reply.started":"2023-07-08T13:38:39.079986Z","shell.execute_reply":"2023-07-08T13:38:39.223222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_file = '/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json'","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:38:46.559964Z","iopub.execute_input":"2023-07-08T13:38:46.560476Z","iopub.status.idle":"2023-07-08T13:38:46.566174Z","shell.execute_reply.started":"2023-07-08T13:38:46.560444Z","shell.execute_reply":"2023-07-08T13:38:46.565233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get access to metadata (lazy evaluation)\ndef get_metadata():\n    with open(my_file, 'r') as f:\n        for line in f:\n            yield line\n            \nmetadata = get_metadata()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:38:50.907032Z","iopub.execute_input":"2023-07-08T13:38:50.907562Z","iopub.status.idle":"2023-07-08T13:38:50.917048Z","shell.execute_reply.started":"2023-07-08T13:38:50.907528Z","shell.execute_reply":"2023-07-08T13:38:50.915726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at titles and abstracts of first few papers\nfor ind, paper in enumerate(metadata):\n    paper = json.loads(paper)\n    print(ind)\n    print(paper['title'])\n    print(paper['abstract'])\n    if (ind == 4):\n        break","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:38:56.039059Z","iopub.execute_input":"2023-07-08T13:38:56.039594Z","iopub.status.idle":"2023-07-08T13:38:56.052266Z","shell.execute_reply.started":"2023-07-08T13:38:56.039559Z","shell.execute_reply":"2023-07-08T13:38:56.050963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check full structure of a paper record\npaper","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:39:01.659238Z","iopub.execute_input":"2023-07-08T13:39:01.659752Z","iopub.status.idle":"2023-07-08T13:39:01.668897Z","shell.execute_reply.started":"2023-07-08T13:39:01.659719Z","shell.execute_reply":"2023-07-08T13:39:01.667355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = ''\nfor ind, paper in tqdm(enumerate(metadata), total=1747307):\n    paper = json.loads(paper)\n    add_txt = paper['title']\n    text = text + add_txt","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:39:12.623169Z","iopub.execute_input":"2023-07-08T13:39:12.624130Z","iopub.status.idle":"2023-07-08T13:40:28.621084Z","shell.execute_reply.started":"2023-07-08T13:39:12.624095Z","shell.execute_reply":"2023-07-08T13:40:28.619884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# number of papers\nprint('Number of papers: ', ind)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:40:32.025080Z","iopub.execute_input":"2023-07-08T13:40:32.025572Z","iopub.status.idle":"2023-07-08T13:40:32.031651Z","shell.execute_reply.started":"2023-07-08T13:40:32.025539Z","shell.execute_reply":"2023-07-08T13:40:32.030687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check lengths of all titles\nprint('Length of combined title texts: ', len(text))","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:40:37.542536Z","iopub.execute_input":"2023-07-08T13:40:37.542990Z","iopub.status.idle":"2023-07-08T13:40:37.550084Z","shell.execute_reply.started":"2023-07-08T13:40:37.542959Z","shell.execute_reply":"2023-07-08T13:40:37.548273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# wordcloud of all paper titles\nstopwords = set(STOPWORDS)\nt1 = time.time()\nwordcloud = WordCloud(stopwords=stopwords, max_font_size=50, max_words=500,\n                      width = 600, height = 400,\n                      background_color=\"black\").generate(text)\nplt.figure(figsize=(12,8))\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis(\"off\")\nplt.show()\nt2 = time.time()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:40:41.881380Z","iopub.execute_input":"2023-07-08T13:40:41.881833Z","iopub.status.idle":"2023-07-08T13:43:25.170536Z","shell.execute_reply.started":"2023-07-08T13:40:41.881802Z","shell.execute_reply":"2023-07-08T13:43:25.169003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Elapsed time for word cloud: ', np.round(t2 - t1,2), 'secs')","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:43:31.439240Z","iopub.execute_input":"2023-07-08T13:43:31.440566Z","iopub.status.idle":"2023-07-08T13:43:31.448142Z","shell.execute_reply.started":"2023-07-08T13:43:31.440522Z","shell.execute_reply":"2023-07-08T13:43:31.446640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = get_metadata()\ntext = ''\ncnt = 0\nmy_cat = 'stat.ML' # define category of interest\nfor ind, paper in tqdm(enumerate(metadata), total=1747307):\n    paper = json.loads(paper)\n    cats = paper['categories']\n    if (cats.find(my_cat)>=0):\n        # print(cats) # for debugging\n        add_txt = paper['title']\n        text = text + add_txt\n        cnt = cnt + 1","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:43:37.554368Z","iopub.execute_input":"2023-07-08T13:43:37.554851Z","iopub.status.idle":"2023-07-08T13:44:31.321045Z","shell.execute_reply.started":"2023-07-08T13:43:37.554816Z","shell.execute_reply":"2023-07-08T13:44:31.319509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# number of papers\nprint('Number of papers: ', cnt)","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:45:04.960895Z","iopub.execute_input":"2023-07-08T13:45:04.961478Z","iopub.status.idle":"2023-07-08T13:45:04.968618Z","shell.execute_reply.started":"2023-07-08T13:45:04.961429Z","shell.execute_reply":"2023-07-08T13:45:04.967379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check lengths of all titles\nprint('Length of combined title texts: ', len(text))","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:45:09.457980Z","iopub.execute_input":"2023-07-08T13:45:09.459203Z","iopub.status.idle":"2023-07-08T13:45:09.465744Z","shell.execute_reply.started":"2023-07-08T13:45:09.459155Z","shell.execute_reply":"2023-07-08T13:45:09.464456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# wordcloud of selected paper titles\nstopwords = set(STOPWORDS)\nt1 = time.time()\nwordcloud = WordCloud(stopwords=stopwords, max_font_size=50, max_words=500,\n                      width = 600, height = 400,\n                      background_color=\"black\").generate(text)\nplt.figure(figsize=(12,8))\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis(\"off\")\nplt.show()\nt2 = time.time()","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:45:14.239445Z","iopub.execute_input":"2023-07-08T13:45:14.240623Z","iopub.status.idle":"2023-07-08T13:45:19.836782Z","shell.execute_reply.started":"2023-07-08T13:45:14.240584Z","shell.execute_reply":"2023-07-08T13:45:19.835482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nr1 = ['essay_category', 'Kaggle competitions', ]\nr2 = ['essay_url', 'https://www.kaggle.com/code/bacharacherif/arxiv-evaluation-papers/edit/run/133944836']\nr3 = ['feedback1_url', 'http://www.kaggle.com/.../your_1st_peer_feedback', ]\nr4 = ['feedback2_url', 'http://www.kaggle.com/.../your_2nd_peer_feedback', ]\nr5 = ['feedback3_url', 'http://www.kaggle.com/.../your_3rd_peer_feedback', ]\nsubmission = pd.DataFrame( [r1,r2,r3,r4,r5], columns=['type','value'])\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-07-08T13:45:25.861430Z","iopub.execute_input":"2023-07-08T13:45:25.861843Z","iopub.status.idle":"2023-07-08T13:45:25.879114Z","shell.execute_reply.started":"2023-07-08T13:45:25.861810Z","shell.execute_reply":"2023-07-08T13:45:25.877806Z"},"trusted":true},"execution_count":null,"outputs":[]}]}