{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-23T07:45:31.798388Z","iopub.execute_input":"2023-04-23T07:45:31.799101Z","iopub.status.idle":"2023-04-23T07:45:31.816960Z","shell.execute_reply.started":"2023-04-23T07:45:31.799050Z","shell.execute_reply":"2023-04-23T07:45:31.815933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/uniprot-description/uniprot_description.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:45:31.820675Z","iopub.execute_input":"2023-04-23T07:45:31.820977Z","iopub.status.idle":"2023-04-23T07:46:00.687879Z","shell.execute_reply.started":"2023-04-23T07:45:31.820950Z","shell.execute_reply":"2023-04-23T07:46:00.686628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:46:00.689349Z","iopub.execute_input":"2023-04-23T07:46:00.690441Z","iopub.status.idle":"2023-04-23T07:46:00.728162Z","shell.execute_reply.started":"2023-04-23T07:46:00.690380Z","shell.execute_reply":"2023-04-23T07:46:00.726970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:46:00.731159Z","iopub.execute_input":"2023-04-23T07:46:00.731576Z","iopub.status.idle":"2023-04-23T07:46:00.738382Z","shell.execute_reply.started":"2023-04-23T07:46:00.731538Z","shell.execute_reply":"2023-04-23T07:46:00.737039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"description\"].replace('', np.nan, inplace=True)\n\ndf[\"description\"].replace('nan', np.nan, inplace=True)\n\ndf = df[df[\"description\"].notna()]\n\ndf = df[df[\"description\"].notnull()]","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:46:00.740334Z","iopub.execute_input":"2023-04-23T07:46:00.740728Z","iopub.status.idle":"2023-04-23T07:46:00.961858Z","shell.execute_reply.started":"2023-04-23T07:46:00.740691Z","shell.execute_reply":"2023-04-23T07:46:00.960581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:46:00.963841Z","iopub.execute_input":"2023-04-23T07:46:00.964243Z","iopub.status.idle":"2023-04-23T07:46:00.970945Z","shell.execute_reply.started":"2023-04-23T07:46:00.964203Z","shell.execute_reply":"2023-04-23T07:46:00.969850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:46:25.558737Z","iopub.execute_input":"2023-04-23T07:46:25.559752Z","iopub.status.idle":"2023-04-23T07:46:25.663489Z","shell.execute_reply.started":"2023-04-23T07:46:25.559695Z","shell.execute_reply":"2023-04-23T07:46:25.662443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install Cython\n!pip install umap-learn\n!pip install --upgrade git+https://github.com/scikit-learn-contrib/hdbscan.git","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:46:39.744034Z","iopub.execute_input":"2023-04-23T07:46:39.744963Z","iopub.status.idle":"2023-04-23T07:48:31.972037Z","shell.execute_reply.started":"2023-04-23T07:46:39.744910Z","shell.execute_reply":"2023-04-23T07:48:31.970565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:48:31.975002Z","iopub.execute_input":"2023-04-23T07:48:31.975533Z","iopub.status.idle":"2023-04-23T07:48:31.983606Z","shell.execute_reply.started":"2023-04-23T07:48:31.975472Z","shell.execute_reply":"2023-04-23T07:48:31.982063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus = df['description'].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:48:36.051433Z","iopub.execute_input":"2023-04-23T07:48:36.051894Z","iopub.status.idle":"2023-04-23T07:48:36.076722Z","shell.execute_reply.started":"2023-04-23T07:48:36.051854Z","shell.execute_reply":"2023-04-23T07:48:36.075233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install sentence_transformers\n!pip install faiss-gpu","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:48:36.078274Z","iopub.execute_input":"2023-04-23T07:48:36.078957Z","iopub.status.idle":"2023-04-23T07:49:01.657114Z","shell.execute_reply.started":"2023-04-23T07:48:36.078918Z","shell.execute_reply":"2023-04-23T07:49:01.655495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer, util\nimport faiss","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:49:01.663037Z","iopub.execute_input":"2023-04-23T07:49:01.663467Z","iopub.status.idle":"2023-04-23T07:49:06.513569Z","shell.execute_reply.started":"2023-04-23T07:49:01.663405Z","shell.execute_reply":"2023-04-23T07:49:06.512488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SentenceTransformer('all-distilroberta-v1')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:49:27.034546Z","iopub.execute_input":"2023-04-23T07:49:27.035189Z","iopub.status.idle":"2023-04-23T07:49:36.453701Z","shell.execute_reply.started":"2023-04-23T07:49:27.035149Z","shell.execute_reply":"2023-04-23T07:49:36.452532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeddings = model.encode(corpus, show_progress_bar=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:49:52.288550Z","iopub.execute_input":"2023-04-23T07:49:52.289596Z","iopub.status.idle":"2023-04-23T07:54:28.360850Z","shell.execute_reply.started":"2023-04-23T07:49:52.289551Z","shell.execute_reply":"2023-04-23T07:54:28.359656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import umap.umap_ as umap","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:54:49.047642Z","iopub.execute_input":"2023-04-23T07:54:49.048628Z","iopub.status.idle":"2023-04-23T07:55:11.347027Z","shell.execute_reply.started":"2023-04-23T07:54:49.048578Z","shell.execute_reply":"2023-04-23T07:55:11.345657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"umap_embedding = umap.UMAP(\n                                 n_neighbors=150,\n                                 min_dist=0,\n                                 n_components=2,\n                                 random_state=23,\n                                 repulsion_strength=1.0,).fit_transform(embeddings)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T07:55:52.466805Z","iopub.execute_input":"2023-04-23T07:55:52.467215Z","iopub.status.idle":"2023-04-23T08:16:01.520713Z","shell.execute_reply.started":"2023-04-23T07:55:52.467177Z","shell.execute_reply":"2023-04-23T08:16:01.519465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import hdbscan","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:16:01.523695Z","iopub.execute_input":"2023-04-23T08:16:01.524114Z","iopub.status.idle":"2023-04-23T08:16:01.594022Z","shell.execute_reply.started":"2023-04-23T08:16:01.524073Z","shell.execute_reply":"2023-04-23T08:16:01.592491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = hdbscan.HDBSCAN(\nmin_samples=1,\nmin_cluster_size=10,\n).fit_predict(umap_embedding)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:16:01.596455Z","iopub.execute_input":"2023-04-23T08:16:01.597250Z","iopub.status.idle":"2023-04-23T08:16:09.159176Z","shell.execute_reply.started":"2023-04-23T08:16:01.597207Z","shell.execute_reply":"2023-04-23T08:16:09.157745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bokeh.plotting import figure, show, output_notebook\nfrom bokeh.models import HoverTool, ColumnDataSource\nfrom bokeh.palettes import Spectral10\nfrom bokeh.models import LinearColorMapper\nfrom bokeh.models import ColorBar\n\noutput_notebook()\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:16:09.162341Z","iopub.execute_input":"2023-04-23T08:16:09.163212Z","iopub.status.idle":"2023-04-23T08:16:09.508381Z","shell.execute_reply.started":"2023-04-23T08:16:09.163157Z","shell.execute_reply":"2023-04-23T08:16:09.507227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = pd.DataFrame(umap_embedding, columns=('x', 'y'))\ntemp['group'] = [int(x) for x in labels]\ntemp['topic'] = df.primaryAccession\n\ngroup = list(temp['group'].unique())\n\n\ndatasource = ColumnDataSource(temp)\n\nyfig = figure(\n    plot_width=600,\n    plot_height=600,\n    tools=('pan, wheel_zoom, reset')\n)\n\nyfig.add_tools(HoverTool(tooltips=\"\"\"\n<div>\n    <div>\n        <span style='font-size: 16px; color: #224499'>Group: </span>\n        <span style='font-size: 18px'>@group</span>\n    </div>\n    <div>\n        <span style='font-size: 16px; color: #224499'>Topic: </span>\n        <span style='font-size: 18px'>@topic</span>\n    </div>\n</div>\n\"\"\"))\n\nexp_cmap = LinearColorMapper(palette='Turbo256', low=min(group), high=max(group))\n    \nyfig.circle(\n    'x',\n    'y',\n    source=datasource,\n    color={'field': 'group', 'transform': exp_cmap},\n    line_alpha=0.6,\n    fill_alpha=0.6,\n    size=4\n)\n\nbar = ColorBar(color_mapper=exp_cmap, location=(0,0))\nyfig.add_layout(bar, \"left\")\n\nshow(yfig)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:34:24.396917Z","iopub.execute_input":"2023-04-23T08:34:24.398032Z","iopub.status.idle":"2023-04-23T08:34:25.069395Z","shell.execute_reply.started":"2023-04-23T08:34:24.397986Z","shell.execute_reply":"2023-04-23T08:34:25.067647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corpus = pd.DataFrame(corpus, columns = ['description'])","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:34:41.814485Z","iopub.execute_input":"2023-04-23T08:34:41.815607Z","iopub.status.idle":"2023-04-23T08:34:41.837327Z","shell.execute_reply.started":"2023-04-23T08:34:41.815565Z","shell.execute_reply":"2023-04-23T08:34:41.836294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list = pd.DataFrame(np.column_stack([df_corpus, labels]), \n                               columns=['sentence', 'cluster'])","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:34:57.079579Z","iopub.execute_input":"2023-04-23T08:34:57.080655Z","iopub.status.idle":"2023-04-23T08:34:57.099128Z","shell.execute_reply.started":"2023-04-23T08:34:57.080612Z","shell.execute_reply":"2023-04-23T08:34:57.097884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_list = list(clustered_list['sentence'])","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:35:20.074215Z","iopub.execute_input":"2023-04-23T08:35:20.075472Z","iopub.status.idle":"2023-04-23T08:35:20.100004Z","shell.execute_reply.started":"2023-04-23T08:35:20.075406Z","shell.execute_reply":"2023-04-23T08:35:20.098837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df['description'], df['primaryAccession']))","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:35:51.311253Z","iopub.execute_input":"2023-04-23T08:35:51.312059Z","iopub.status.idle":"2023-04-23T08:35:51.396092Z","shell.execute_reply.started":"2023-04-23T08:35:51.312013Z","shell.execute_reply":"2023-04-23T08:35:51.394672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list['primaryAccession'] = [dict_lookup[item] for item in key_list]","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:36:09.286644Z","iopub.execute_input":"2023-04-23T08:36:09.287399Z","iopub.status.idle":"2023-04-23T08:36:09.341444Z","shell.execute_reply.started":"2023-04-23T08:36:09.287360Z","shell.execute_reply":"2023-04-23T08:36:09.340316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list['cluster'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:36:16.884660Z","iopub.execute_input":"2023-04-23T08:36:16.885745Z","iopub.status.idle":"2023-04-23T08:36:16.905943Z","shell.execute_reply.started":"2023-04-23T08:36:16.885684Z","shell.execute_reply":"2023-04-23T08:36:16.904531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cluster_worldcloud(cluster_num):\n    comment_words = ''\n    stopwords = set(STOPWORDS)\n\n \n    for val in clustered_list[clustered_list['cluster']==cluster_num].sentence.values:\n      \n      \n        val = str(val)\n\n      \n        tokens = val.split()\n      \n      \n        for i in range(len(tokens)):\n            tokens[i] = tokens[i].lower()\n      \n        comment_words += \" \".join(tokens)+\" \"\n\n    wordcloud = WordCloud( background_color = 'white',\n                          stopwords = stopwords, \n                          width = 2048, height = 1080).generate(comment_words)\n\n\n                     \n    plt.figure(figsize= (20,10) )\n    plt.imshow(wordcloud, interpolation='bilinear')\n    plt.axis(\"off\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:40:58.723380Z","iopub.execute_input":"2023-04-23T08:40:58.724178Z","iopub.status.idle":"2023-04-23T08:40:58.733334Z","shell.execute_reply.started":"2023-04-23T08:40:58.724135Z","shell.execute_reply":"2023-04-23T08:40:58.731872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport seaborn as sns\nimport nltk\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nimport matplotlib.pyplot as plt\n%matplotlib inline\nsns.set()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:37:45.817192Z","iopub.execute_input":"2023-04-23T08:37:45.818024Z","iopub.status.idle":"2023-04-23T08:37:45.827412Z","shell.execute_reply.started":"2023-04-23T08:37:45.817978Z","shell.execute_reply":"2023-04-23T08:37:45.826177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster_worldcloud(3782)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T08:41:01.359206Z","iopub.execute_input":"2023-04-23T08:41:01.360035Z","iopub.status.idle":"2023-04-23T08:41:05.257966Z","shell.execute_reply.started":"2023-04-23T08:41:01.359983Z","shell.execute_reply":"2023-04-23T08:41:05.256754Z"},"trusted":true},"execution_count":null,"outputs":[]}]}