{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Arxiv Metadata Analysis\n\n**If you find this notebook useful, please upvote.**\n\n2023-07-02: extraction of paper title embeddings, reduce the embeddings do 2D and cluster them and visualize.","metadata":{}},{"cell_type":"code","source":"!pip install transformers","metadata":{"execution":{"iopub.status.busy":"2023-07-02T03:40:12.447630Z","iopub.execute_input":"2023-07-02T03:40:12.448027Z","iopub.status.idle":"2023-07-02T03:40:25.408138Z","shell.execute_reply.started":"2023-07-02T03:40:12.448000Z","shell.execute_reply":"2023-07-02T03:40:25.406838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport umap\nfrom sklearn.cluster import KMeans\nfrom transformers import BertTokenizer, BertModel\nfrom tqdm import tqdm\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-02T03:40:25.413977Z","iopub.execute_input":"2023-07-02T03:40:25.414332Z","iopub.status.idle":"2023-07-02T03:40:41.012841Z","shell.execute_reply.started":"2023-07-02T03:40:25.414281Z","shell.execute_reply":"2023-07-02T03:40:41.011800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\narxiv_metadata_all = pd.read_json(\n    \"/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json\", \n    lines=True, convert_dates=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T03:40:41.014205Z","iopub.execute_input":"2023-07-02T03:40:41.014635Z","iopub.status.idle":"2023-07-02T03:42:57.645854Z","shell.execute_reply.started":"2023-07-02T03:40:41.014597Z","shell.execute_reply":"2023-07-02T03:42:57.644681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluate only cs.AI \narxiv_metadata = arxiv_metadata_all[\n    arxiv_metadata_all['categories'].str.contains('cs.AI')]\n# only 2021 to 2023 articles \narxiv_metadata = arxiv_metadata[arxiv_metadata.update_date >= '2021-01-01']\n\nlen(arxiv_metadata)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T03:42:57.659734Z","iopub.execute_input":"2023-07-02T03:42:57.660047Z","iopub.status.idle":"2023-07-02T03:42:59.841483Z","shell.execute_reply.started":"2023-07-02T03:42:57.660022Z","shell.execute_reply":"2023-07-02T03:42:59.840438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# consider only the articles whose title contains any of the keywords\nkeywords = [\n    'generative', 'diffusion', 'gpt', 'gpt-4', 'large language'] \narxiv_metadata = arxiv_metadata[\n    arxiv_metadata['title'].str.contains('|'.join(keywords), case=False)]\n\nprint(len(arxiv_metadata))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-02T03:42:59.843076Z","iopub.execute_input":"2023-07-02T03:42:59.843540Z","iopub.status.idle":"2023-07-02T03:43:00.173967Z","shell.execute_reply.started":"2023-07-02T03:42:59.843502Z","shell.execute_reply":"2023-07-02T03:43:00.172759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print first 32 titles\ntitles = [title for title in arxiv_metadata['title'].str.lstrip()]\nfor title in titles[:2**5]:\n    print(f'* {title}')","metadata":{"execution":{"iopub.status.busy":"2023-07-02T03:43:00.175414Z","iopub.execute_input":"2023-07-02T03:43:00.175838Z","iopub.status.idle":"2023-07-02T03:43:00.184329Z","shell.execute_reply.started":"2023-07-02T03:43:00.175801Z","shell.execute_reply":"2023-07-02T03:43:00.183222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's play with the embeddings of the paper titles.","metadata":{}},{"cell_type":"code","source":"# Load pre-trained model and tokenizer\nmodel = BertModel.from_pretrained('bert-base-uncased')\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n# generate the embeddings\ntitle_embeddings = []\nfor title in tqdm(titles):\n    input_ids = tokenizer.encode(title, return_tensors='pt')\n    # Extract embeddings\n    with torch.no_grad():\n        outputs = model(input_ids)\n        embeddings = outputs.last_hidden_state[0]\n\n    # Compute average embedding for the entire sentence\n    title_embedding = torch.mean(embeddings, dim=0)\n    title_embeddings.append(title_embedding)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T03:43:00.185685Z","iopub.execute_input":"2023-07-02T03:43:00.185983Z","iopub.status.idle":"2023-07-02T03:44:11.500713Z","shell.execute_reply.started":"2023-07-02T03:43:00.185959Z","shell.execute_reply":"2023-07-02T03:44:11.499413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert list of title embeddings to a PyTorch tensor\n\ntitle_embeddings = torch.stack(title_embeddings)\n\n# Create a UMAP instance\nreducer = umap.UMAP()\n\n# Fit the UMAP model to the title embeddings\nembedding = reducer.fit_transform(title_embeddings)\n\n# Plot the resulting 2D embedding\nplt.scatter(embedding[:, 0], embedding[:, 1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T03:44:11.502380Z","iopub.execute_input":"2023-07-02T03:44:11.503202Z","iopub.status.idle":"2023-07-02T03:44:24.539083Z","shell.execute_reply.started":"2023-07-02T03:44:11.503159Z","shell.execute_reply":"2023-07-02T03:44:24.537935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cluster the UMAP-reduced embeddings using k-means\nN_CLUSTERS = 20\nkmeans = KMeans(n_clusters=N_CLUSTERS)\nclusters = kmeans.fit_predict(embedding)\n\n# Plot the UMAP embeddings with different colors for each cluster\nplt.scatter(embedding[:, 0], embedding[:, 1], c=clusters, cmap='tab20')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T04:09:24.598955Z","iopub.execute_input":"2023-07-02T04:09:24.600666Z","iopub.status.idle":"2023-07-02T04:09:24.995751Z","shell.execute_reply.started":"2023-07-02T04:09:24.600615Z","shell.execute_reply":"2023-07-02T04:09:24.994525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a DataFrame with the titles and cluster assignments\ndf = pd.DataFrame({'title': titles, 'cluster': clusters})\n\n# Group the DataFrame by cluster and print the first 20 titles in each cluster\nN_TITLES = 20\nfor cluster, group in df.groupby('cluster'):\n    print(f'** Cluster {cluster}:')\n    for t in (group['title'].to_list()[:N_TITLES]):\n        print(f'* {t}')\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T03:50:19.675681Z","iopub.execute_input":"2023-07-02T03:50:19.676130Z","iopub.status.idle":"2023-07-02T03:50:19.695278Z","shell.execute_reply.started":"2023-07-02T03:50:19.676093Z","shell.execute_reply":"2023-07-02T03:50:19.693653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}