{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd # data processing\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-15T14:09:50.065914Z","iopub.execute_input":"2023-05-15T14:09:50.066465Z","iopub.status.idle":"2023-05-15T14:09:50.082494Z","shell.execute_reply.started":"2023-05-15T14:09:50.066426Z","shell.execute_reply":"2023-05-15T14:09:50.080797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_writeups = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv\", \n                              parse_dates=[0, 3])\nkaggle_writeups.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:09:50.085309Z","iopub.execute_input":"2023-05-15T14:09:50.085747Z","iopub.status.idle":"2023-05-15T14:09:50.856653Z","shell.execute_reply.started":"2023-05-15T14:09:50.085711Z","shell.execute_reply":"2023-05-15T14:09:50.855273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_writeups.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:09:50.858284Z","iopub.execute_input":"2023-05-15T14:09:50.858648Z","iopub.status.idle":"2023-05-15T14:09:50.878655Z","shell.execute_reply.started":"2023-05-15T14:09:50.858617Z","shell.execute_reply":"2023-05-15T14:09:50.876683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_writeups = kaggle_writeups[kaggle_writeups['Competition Launch Date'] >= '2020-01-01']\nkaggle_writeups.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:09:50.883005Z","iopub.execute_input":"2023-05-15T14:09:50.883468Z","iopub.status.idle":"2023-05-15T14:09:50.906472Z","shell.execute_reply.started":"2023-05-15T14:09:50.883435Z","shell.execute_reply":"2023-05-15T14:09:50.905142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_writeups['Title of Competition'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:09:50.908366Z","iopub.execute_input":"2023-05-15T14:09:50.90874Z","iopub.status.idle":"2023-05-15T14:09:50.918045Z","shell.execute_reply.started":"2023-05-15T14:09:50.908709Z","shell.execute_reply":"2023-05-15T14:09:50.916209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_writeups.loc[2054].Writeup","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:09:50.920094Z","iopub.execute_input":"2023-05-15T14:09:50.920481Z","iopub.status.idle":"2023-05-15T14:09:50.93415Z","shell.execute_reply.started":"2023-05-15T14:09:50.920448Z","shell.execute_reply":"2023-05-15T14:09:50.932488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"language_comps = kaggle_writeups[(kaggle_writeups.Writeup.str.contains(\n    'language') | kaggle_writeups['Title of Competition'].str.contains(\n    'language')).fillna(False)]['Title of Competition'].unique()\nprint(f\"How many now: {len(language_comps)}\")\nlanguage_comps","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:09:50.93587Z","iopub.execute_input":"2023-05-15T14:09:50.936436Z","iopub.status.idle":"2023-05-15T14:09:50.964345Z","shell.execute_reply.started":"2023-05-15T14:09:50.936388Z","shell.execute_reply":"2023-05-15T14:09:50.962158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kaggle_writeups[kaggle_writeups['Title of Competition'].isin(language_comps\n                                                            )].to_csv('kaggle_writeups_filtered.csv', \n                                                                      index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:09:50.965908Z","iopub.execute_input":"2023-05-15T14:09:50.966479Z","iopub.status.idle":"2023-05-15T14:09:51.014552Z","shell.execute_reply.started":"2023-05-15T14:09:50.966429Z","shell.execute_reply":"2023-05-15T14:09:51.012375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arxiv_metadata = pd.read_json(\"/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json\", \n                              lines=True,\n                             convert_dates=True)\narxiv_metadata.head() ","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:09:51.016436Z","iopub.execute_input":"2023-05-15T14:09:51.017005Z","iopub.status.idle":"2023-05-15T14:11:55.392411Z","shell.execute_reply.started":"2023-05-15T14:09:51.016956Z","shell.execute_reply":"2023-05-15T14:11:55.39107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arxiv_metadata.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:55.39531Z","iopub.execute_input":"2023-05-15T14:11:55.395804Z","iopub.status.idle":"2023-05-15T14:11:55.409321Z","shell.execute_reply.started":"2023-05-15T14:11:55.395754Z","shell.execute_reply":"2023-05-15T14:11:55.40812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arxiv_metadata = arxiv_metadata[arxiv_metadata.update_date >= '2021-01-01']\narxiv_metadata.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:55.410706Z","iopub.execute_input":"2023-05-15T14:11:55.412204Z","iopub.status.idle":"2023-05-15T14:11:58.114646Z","shell.execute_reply.started":"2023-05-15T14:11:55.412166Z","shell.execute_reply":"2023-05-15T14:11:58.111967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat = arxiv_metadata.categories.unique()\nprint(f\"How many categories are there {len(cat)}\")\ncat[:20]","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:58.117403Z","iopub.execute_input":"2023-05-15T14:11:58.117895Z","iopub.status.idle":"2023-05-15T14:11:58.393087Z","shell.execute_reply.started":"2023-05-15T14:11:58.117852Z","shell.execute_reply":"2023-05-15T14:11:58.390853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arxiv_metadata = arxiv_metadata[arxiv_metadata['categories'].str.contains(\n    'cs.CL') & arxiv_metadata['categories'].str.contains('cs.AI')]\narxiv_metadata.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:58.398576Z","iopub.execute_input":"2023-05-15T14:11:58.400355Z","iopub.status.idle":"2023-05-15T14:11:59.327336Z","shell.execute_reply.started":"2023-05-15T14:11:58.400278Z","shell.execute_reply":"2023-05-15T14:11:59.324676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arxiv_metadata = arxiv_metadata[~arxiv_metadata.doi.isna()]\nprint(f\"How many papers are left now: {len(arxiv_metadata)}\")\narxiv_metadata.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:59.330101Z","iopub.execute_input":"2023-05-15T14:11:59.330675Z","iopub.status.idle":"2023-05-15T14:11:59.369329Z","shell.execute_reply.started":"2023-05-15T14:11:59.330631Z","shell.execute_reply":"2023-05-15T14:11:59.367711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arxiv_metadata.to_csv('filtered_text_arix.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:59.371692Z","iopub.execute_input":"2023-05-15T14:11:59.372223Z","iopub.status.idle":"2023-05-15T14:11:59.40713Z","shell.execute_reply.started":"2023-05-15T14:11:59.372171Z","shell.execute_reply":"2023-05-15T14:11:59.404725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/sample_submission.csv\")\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:59.409386Z","iopub.execute_input":"2023-05-15T14:11:59.409895Z","iopub.status.idle":"2023-05-15T14:11:59.433419Z","shell.execute_reply.started":"2023-05-15T14:11:59.409865Z","shell.execute_reply":"2023-05-15T14:11:59.431525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.loc[0]['value'] = 'Text data'\nsubmission.loc[1]['value'] = 'https://www.kaggle.com/code/anthony35813/ai-report-a-first-look-at-the-data-for-text-data'\nsubmission.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:59.435267Z","iopub.execute_input":"2023-05-15T14:11:59.43618Z","iopub.status.idle":"2023-05-15T14:11:59.448656Z","shell.execute_reply.started":"2023-05-15T14:11:59.436142Z","shell.execute_reply":"2023-05-15T14:11:59.447716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:11:59.450647Z","iopub.execute_input":"2023-05-15T14:11:59.451821Z","iopub.status.idle":"2023-05-15T14:11:59.468126Z","shell.execute_reply.started":"2023-05-15T14:11:59.451743Z","shell.execute_reply":"2023-05-15T14:11:59.466884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion  \n\n\n","metadata":{}}]}