{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-23T10:58:42.299238Z","iopub.execute_input":"2023-04-23T10:58:42.300366Z","iopub.status.idle":"2023-04-23T10:58:42.315318Z","shell.execute_reply.started":"2023-04-23T10:58:42.300322Z","shell.execute_reply":"2023-04-23T10:58:42.314155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/uniprot-description-only/uniprot_description_only.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-23T10:58:42.318599Z","iopub.execute_input":"2023-04-23T10:58:42.319355Z","iopub.status.idle":"2023-04-23T10:58:43.297425Z","shell.execute_reply.started":"2023-04-23T10:58:42.319315Z","shell.execute_reply":"2023-04-23T10:58:43.296344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-23T10:58:43.298777Z","iopub.execute_input":"2023-04-23T10:58:43.299131Z","iopub.status.idle":"2023-04-23T10:58:43.310181Z","shell.execute_reply.started":"2023-04-23T10:58:43.299095Z","shell.execute_reply":"2023-04-23T10:58:43.309100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-23T10:58:43.311529Z","iopub.execute_input":"2023-04-23T10:58:43.312483Z","iopub.status.idle":"2023-04-23T10:58:43.321943Z","shell.execute_reply.started":"2023-04-23T10:58:43.312445Z","shell.execute_reply":"2023-04-23T10:58:43.320671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"description\"].replace('', np.nan, inplace=True)\n\ndf[\"description\"].replace('nan', np.nan, inplace=True)\n\ndf = df[df[\"description\"].notna()]\n\ndf = df[df[\"description\"].notnull()]","metadata":{"execution":{"iopub.status.busy":"2023-04-23T10:58:43.325431Z","iopub.execute_input":"2023-04-23T10:58:43.326104Z","iopub.status.idle":"2023-04-23T10:58:43.388977Z","shell.execute_reply.started":"2023-04-23T10:58:43.326068Z","shell.execute_reply":"2023-04-23T10:58:43.387929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-23T10:58:43.390386Z","iopub.execute_input":"2023-04-23T10:58:43.391058Z","iopub.status.idle":"2023-04-23T10:58:43.398766Z","shell.execute_reply.started":"2023-04-23T10:58:43.391015Z","shell.execute_reply":"2023-04-23T10:58:43.397669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T10:58:43.400472Z","iopub.execute_input":"2023-04-23T10:58:43.401130Z","iopub.status.idle":"2023-04-23T10:58:43.415123Z","shell.execute_reply.started":"2023-04-23T10:58:43.401093Z","shell.execute_reply":"2023-04-23T10:58:43.414158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install Cython\n!pip install umap-learn\n!pip install --upgrade git+https://github.com/scikit-learn-contrib/hdbscan.git","metadata":{"execution":{"iopub.status.busy":"2023-04-23T10:58:43.418228Z","iopub.execute_input":"2023-04-23T10:58:43.418505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom tqdm import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus = df['description'].tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install sentence_transformers","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer, util","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SentenceTransformer('all-distilroberta-v1')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nembeddings = model.encode(corpus, show_progress_bar=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import umap.umap_ as umap","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\numap_embedding = umap.UMAP(\n                                 n_neighbors=150,\n                                 min_dist=0,\n                                 n_components=2,\n                                 random_state=23,\n                                 repulsion_strength=1.0,).fit_transform(embeddings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import hdbscan","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nlabels = hdbscan.HDBSCAN(\nmin_samples=1,\nmin_cluster_size=10,\n).fit_predict(umap_embedding)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bokeh.plotting import figure, show, output_notebook\nfrom bokeh.models import HoverTool, ColumnDataSource\nfrom bokeh.palettes import Spectral10\nfrom bokeh.models import LinearColorMapper\nfrom bokeh.models import ColorBar\n\noutput_notebook()\n\n%matplotlib inline","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = pd.DataFrame(umap_embedding, columns=('x', 'y'))\ntemp['group'] = [int(x) for x in labels]\ntemp['topic'] = df.gene_value\n\ngroup = list(temp['group'].unique())\n\n\ndatasource = ColumnDataSource(temp)\n\nyfig = figure(\n    plot_width=600,\n    plot_height=600,\n    tools=('pan, wheel_zoom, reset')\n)\n\nyfig.add_tools(HoverTool(tooltips=\"\"\"\n<div>\n    <div>\n        <span style='font-size: 16px; color: #224499'>Group: </span>\n        <span style='font-size: 18px'>@group</span>\n    </div>\n    <div>\n        <span style='font-size: 16px; color: #224499'>Topic: </span>\n        <span style='font-size: 18px'>@topic</span>\n    </div>\n</div>\n\"\"\"))\n\nexp_cmap = LinearColorMapper(palette='Turbo256', low=min(group), high=max(group))\n    \nyfig.circle(\n    'x',\n    'y',\n    source=datasource,\n    color={'field': 'group', 'transform': exp_cmap},\n    line_alpha=0.6,\n    fill_alpha=0.6,\n    size=4\n)\n\nbar = ColorBar(color_mapper=exp_cmap, location=(0,0))\nyfig.add_layout(bar, \"left\")\n\nshow(yfig)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corpus = pd.DataFrame(corpus, columns = ['description'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list = pd.DataFrame(np.column_stack([df_corpus, labels]), \n                               columns=['sentence', 'cluster'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_list = list(clustered_list['sentence'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df['description'], df['gene_value']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list['gene_value'] = [dict_lookup[item] for item in key_list]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list['cluster'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cluster_worldcloud(cluster_num):\n    comment_words = ''\n    stopwords = set(STOPWORDS)\n\n \n    for val in clustered_list[clustered_list['cluster']==cluster_num].sentence.values:\n      \n      \n        val = str(val)\n\n      \n        tokens = val.split()\n      \n      \n        for i in range(len(tokens)):\n            tokens[i] = tokens[i].lower()\n      \n        comment_words += \" \".join(tokens)+\" \"\n\n    wordcloud = WordCloud( background_color = 'white',\n                          stopwords = stopwords, \n                          width = 2048, height = 1080).generate(comment_words)\n\n\n                     \n    plt.figure(figsize= (20,10) )\n    plt.imshow(wordcloud, interpolation='bilinear')\n    plt.axis(\"off\")\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport seaborn as sns\nimport nltk\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nimport matplotlib.pyplot as plt\n%matplotlib inline\nsns.set()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster_worldcloud(4845)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb = pd.DataFrame({'description': corpus, 'embedding': None})\ndf_emb['embedding'] = embeddings.tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_columns = pd.DataFrame(df_emb['embedding'].values.tolist())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new = pd.concat([df_emb, df_emb_columns], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_list = list(df_emb_new['description'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df['description'], df['gene_value']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new['gene_value'] = [dict_lookup[item] for item in key_list]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new = df_emb_new.drop(columns=['description', 'embedding'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = df_emb_new.columns.tolist()\n\ncols = cols[-1:] + cols[:-1]\n\ndf_emb_new = df_emb_new[cols]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new.to_csv('./sentence_transformers_embeddings.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}