{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on May 29, 2023. By Marília Prata, mpwolke.","metadata":{"_kg_hide-input":false}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objs as go\n\nimport plotly\nplotly.offline.init_notebook_mode(connected=True)\n\nfrom pathlib import Path\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-29T21:11:06.661958Z","iopub.execute_input":"2023-05-29T21:11:06.662865Z","iopub.status.idle":"2023-05-29T21:11:06.680819Z","shell.execute_reply.started":"2023-05-29T21:11:06.662831Z","shell.execute_reply":"2023-05-29T21:11:06.679670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rake-nltk","metadata":{"execution":{"iopub.status.busy":"2023-05-29T20:44:16.035822Z","iopub.execute_input":"2023-05-29T20:44:16.036592Z","iopub.status.idle":"2023-05-29T20:44:31.885543Z","shell.execute_reply.started":"2023-05-29T20:44:16.036552Z","shell.execute_reply":"2023-05-29T20:44:31.884241Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom nltk.stem import WordNetLemmatizer,PorterStemmer\nfrom nltk.tokenize import word_tokenize\nfrom sklearn.cluster import DBSCAN\nfrom nltk.corpus import stopwords\nfrom spacy.matcher import Matcher \nfrom collections import  Counter\nimport matplotlib.pyplot as plt\nfrom spacy.tokens import Span \nimport tensorflow_hub as hub\nfrom rake_nltk import Rake\nimport tensorflow as tf\nimport pyLDAvis.gensim\nfrom tqdm import tqdm\nimport seaborn as sns\nimport networkx as nx\nimport pandas as pd\nimport numpy as np\nimport pyLDAvis\nimport gensim\nimport spacy\nimport gc","metadata":{"execution":{"iopub.status.busy":"2023-05-29T20:45:13.959639Z","iopub.execute_input":"2023-05-29T20:45:13.960055Z","iopub.status.idle":"2023-05-29T20:45:27.625437Z","shell.execute_reply.started":"2023-05-29T20:45:13.960021Z","shell.execute_reply":"2023-05-29T20:45:27.624302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path=\"../input/2023-kaggle-ai-report/\"\nall_sources=pd.read_csv(path+\"kaggle_writeups_20230510.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-05-29T20:49:42.712124Z","iopub.execute_input":"2023-05-29T20:49:42.712546Z","iopub.status.idle":"2023-05-29T20:49:43.070835Z","shell.execute_reply.started":"2023-05-29T20:49:42.712518Z","shell.execute_reply":"2023-05-29T20:49:43.069970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_sources.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T20:50:53.572615Z","iopub.execute_input":"2023-05-29T20:50:53.573749Z","iopub.status.idle":"2023-05-29T20:50:53.605309Z","shell.execute_reply.started":"2023-05-29T20:50:53.573699Z","shell.execute_reply":"2023-05-29T20:50:53.604220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Distribution of Title of the Competition length","metadata":{}},{"cell_type":"code","source":"#Shahules https://www.kaggle.com/code/shahules/cord-tools-and-knowledge-graphs#notebook-container\n\nheadline_length=all_sources['Title of Competition'].str.len()\nsns.distplot(headline_length)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T20:51:23.764697Z","iopub.execute_input":"2023-05-29T20:51:23.765105Z","iopub.status.idle":"2023-05-29T20:51:24.204848Z","shell.execute_reply.started":"2023-05-29T20:51:23.765077Z","shell.execute_reply":"2023-05-29T20:51:24.203695Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Distribution of Title of Writeup length","metadata":{}},{"cell_type":"code","source":"headline_length=all_sources['Title of Writeup'].str.len()\nplt.hist(headline_length, color='r')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:04:32.902623Z","iopub.execute_input":"2023-05-29T21:04:32.903771Z","iopub.status.idle":"2023-05-29T21:04:33.098462Z","shell.execute_reply.started":"2023-05-29T21:04:32.903735Z","shell.execute_reply":"2023-05-29T21:04:33.097467Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop=set(stopwords.words('english'))\n\ndef build_list(df,col=\"Title of Competition\"):\n    corpus=[]\n    lem=WordNetLemmatizer()\n    stop=set(stopwords.words('english'))\n    new= df[col].dropna().str.split()\n    new=new.values.tolist()\n    corpus=[lem.lemmatize(word.lower()) for i in new for word in i if(word) not in stop]\n    \n    return corpus","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:06:49.726844Z","iopub.execute_input":"2023-05-29T21:06:49.728145Z","iopub.status.idle":"2023-05-29T21:06:49.742209Z","shell.execute_reply.started":"2023-05-29T21:06:49.728099Z","shell.execute_reply":"2023-05-29T21:06:49.740959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Most common words in Competitions titles","metadata":{}},{"cell_type":"code","source":"corpus=build_list(all_sources)\ncounter=Counter(corpus)\nmost=counter.most_common()\nx=[]\ny=[]\nfor word,count in most[:10]:\n    if (word not in stop) :\n        x.append(word)\n        y.append(count)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:25:46.920370Z","iopub.execute_input":"2023-05-29T21:25:46.921522Z","iopub.status.idle":"2023-05-29T21:25:49.882409Z","shell.execute_reply.started":"2023-05-29T21:25:46.921476Z","shell.execute_reply":"2023-05-29T21:25:49.881500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(9,7))\nsns.barplot(x=y,y=x);","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:26:44.876206Z","iopub.execute_input":"2023-05-29T21:26:44.876636Z","iopub.status.idle":"2023-05-29T21:26:45.172126Z","shell.execute_reply.started":"2023-05-29T21:26:44.876599Z","shell.execute_reply":"2023-05-29T21:26:45.171045Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Most common words in Writeups","metadata":{}},{"cell_type":"code","source":"corpus=build_list(all_sources,\"Title of Writeup\")\ncounter=Counter(corpus)\nmost=counter.most_common()\nx=[]\ny=[]\nfor word,count in most[:10]:\n    if (word not in stop) :\n        x.append(word)\n        y.append(count)\n        \nplt.figure(figsize=(9,7))\nsns.barplot(x=y,y=x);","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:27:47.709181Z","iopub.execute_input":"2023-05-29T21:27:47.709629Z","iopub.status.idle":"2023-05-29T21:27:48.099026Z","shell.execute_reply.started":"2023-05-29T21:27:47.709599Z","shell.execute_reply":"2023-05-29T21:27:48.097945Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Most common bi-grams in Title of Competitions","metadata":{}},{"cell_type":"code","source":"def get_top_ngram(corpus, n=None):\n    vec = CountVectorizer(ngram_range=(n, n)).fit(corpus)\n    bag_of_words = vec.transform(corpus)\n    sum_words = bag_of_words.sum(axis=0) \n    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]\n    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)\n    return words_freq[:10]","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:30:14.615284Z","iopub.execute_input":"2023-05-29T21:30:14.615763Z","iopub.status.idle":"2023-05-29T21:30:14.623094Z","shell.execute_reply.started":"2023-05-29T21:30:14.615729Z","shell.execute_reply":"2023-05-29T21:30:14.622049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_n_bigrams=get_top_ngram(all_sources['Title of Competition'].dropna(),2)[:10]\nx,y=map(list,zip(*top_n_bigrams))\nplt.figure(figsize=(9,7))\nsns.barplot(x=y,y=x);","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:31:49.565446Z","iopub.execute_input":"2023-05-29T21:31:49.565822Z","iopub.status.idle":"2023-05-29T21:31:49.951594Z","shell.execute_reply.started":"2023-05-29T21:31:49.565794Z","shell.execute_reply":"2023-05-29T21:31:49.950513Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Most common tri-grams in Title of Competitions","metadata":{}},{"cell_type":"code","source":"top_tri_grams=get_top_ngram(all_sources['Title of Competition'].dropna(),n=3)\nx,y=map(list,zip(*top_tri_grams))\nplt.figure(figsize=(9,7))\nsns.barplot(x=y,y=x);","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:33:35.500891Z","iopub.execute_input":"2023-05-29T21:33:35.501299Z","iopub.status.idle":"2023-05-29T21:33:35.875537Z","shell.execute_reply.started":"2023-05-29T21:33:35.501268Z","shell.execute_reply":"2023-05-29T21:33:35.874452Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Modeling","metadata":{}},{"cell_type":"code","source":"def preprocess_news(df):\n    corpus=[]\n    stem=PorterStemmer()\n    lem=WordNetLemmatizer()\n    for news in df['Title of Competition'].dropna()[:5000]:\n        words=[w for w in word_tokenize(news) if (w not in stop)]\n        \n        words=[lem.lemmatize(w) for w in words if len(w)>2]\n        \n        corpus.append(words)\n    return corpus","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:35:43.341481Z","iopub.execute_input":"2023-05-29T21:35:43.341894Z","iopub.status.idle":"2023-05-29T21:35:43.349124Z","shell.execute_reply.started":"2023-05-29T21:35:43.341863Z","shell.execute_reply":"2023-05-29T21:35:43.347929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus=preprocess_news(all_sources)\ndic=gensim.corpora.Dictionary(corpus)\nbow_corpus = [dic.doc2bow(doc) for doc in corpus]","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:35:48.480099Z","iopub.execute_input":"2023-05-29T21:35:48.480483Z","iopub.status.idle":"2023-05-29T21:35:49.110368Z","shell.execute_reply.started":"2023-05-29T21:35:48.480441Z","shell.execute_reply":"2023-05-29T21:35:49.109146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lda_model =  gensim.models.LdaMulticore(bow_corpus, \n                                   num_topics = 4, \n                                   id2word = dic,                                    \n                                   passes = 10,\n                                   workers = 2)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:36:08.550894Z","iopub.status.idle":"2023-05-29T21:36:17.589881Z","shell.execute_reply.started":"2023-05-29T21:36:08.551261Z","shell.execute_reply":"2023-05-29T21:36:17.588671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lda_model.show_topics()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:38:06.275824Z","iopub.execute_input":"2023-05-29T21:38:06.276213Z","iopub.status.idle":"2023-05-29T21:38:06.285017Z","shell.execute_reply.started":"2023-05-29T21:38:06.276177Z","shell.execute_reply":"2023-05-29T21:38:06.283893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#pyLDAvis\n\npyLDAvis is designed to help users interpret the topics in a topic model that has been fit to a corpus of text data. The package extracts information from a fitted LDA topic model to inform an interactive web-based visualization.\n\nVisualizing the topic models","metadata":{}},{"cell_type":"code","source":"pyLDAvis.enable_notebook()\nvis = pyLDAvis.gensim.prepare(lda_model, bow_corpus, dic)\nvis","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:39:05.769852Z","iopub.execute_input":"2023-05-29T21:39:05.770694Z","iopub.status.idle":"2023-05-29T21:39:08.873853Z","shell.execute_reply.started":"2023-05-29T21:39:05.770642Z","shell.execute_reply":"2023-05-29T21:39:08.872549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del corpus,top_n_bigrams,lda_model,bow_corpus,top_tri_grams\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:40:31.397402Z","iopub.execute_input":"2023-05-29T21:40:31.397853Z","iopub.status.idle":"2023-05-29T21:40:32.101286Z","shell.execute_reply.started":"2023-05-29T21:40:31.397809Z","shell.execute_reply":"2023-05-29T21:40:32.099943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Find Similar Papers","metadata":{}},{"cell_type":"markdown","source":"#Method 1: Using Universal sentence Encoder\n\nDone using Universal Sentence encoder and cosine similarity.\n\n![](https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTzJnJPFruEXT6T7TXRiGjLJIJHCiCq1ZDdgw&usqp=CAU)youtube","metadata":{}},{"cell_type":"code","source":"def prepare_similarity(vectors):\n    similarity=cosine_similarity(vectors)\n    return similarity\n\ndef get_top_similar(sentence, sentence_list, similarity_matrix, topN):\n    # find the index of sentence in list\n    index = sentence_list.index(sentence)\n    # get the corresponding row in similarity matrix\n    similarity_row = np.array(similarity_matrix[index, :])\n    # get the indices of top similar\n    indices = similarity_row.argsort()[-topN:][::-1]\n    return [(i,sentence_list[i]) for i in indices]","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:46:12.521142Z","iopub.execute_input":"2023-05-29T21:46:12.521581Z","iopub.status.idle":"2023-05-29T21:46:12.529302Z","shell.execute_reply.started":"2023-05-29T21:46:12.521549Z","shell.execute_reply":"2023-05-29T21:46:12.528196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Dataset by Brian\n\nmodule_url = \"../input/universalsentenceencoderlarge4\" \n# Import the Universal Sentence Encoder's TF Hub module\nembed = hub.load(module_url)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:46:33.894452Z","iopub.execute_input":"2023-05-29T21:46:33.894905Z","iopub.status.idle":"2023-05-29T21:46:50.585252Z","shell.execute_reply.started":"2023-05-29T21:46:33.894867Z","shell.execute_reply":"2023-05-29T21:46:50.584266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"titles=all_sources['Title of Competition'].fillna(\"Unknown\")\nembed_vectors=embed(titles[:100].values)['outputs'].numpy()\nsentence_list=titles.values.tolist()\nsentence=titles.iloc[5]\nprint(\"Find similar research papers for :\")\nprint(sentence)\n\nsimilarity_matrix=prepare_similarity(embed_vectors)\nsimilar=get_top_similar(sentence,sentence_list,similarity_matrix,6)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:37:02.626152Z","iopub.execute_input":"2023-05-29T22:37:02.626580Z","iopub.status.idle":"2023-05-29T22:37:03.062914Z","shell.execute_reply.started":"2023-05-29T22:37:02.626547Z","shell.execute_reply":"2023-05-29T22:37:03.061945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Why that RTA Freeway Travel Time Prediction???","metadata":{}},{"cell_type":"code","source":"for sentence in similar:\n    print(sentence)\n    print(\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:48:49.256696Z","iopub.execute_input":"2023-05-29T21:48:49.257101Z","iopub.status.idle":"2023-05-29T21:48:49.262564Z","shell.execute_reply.started":"2023-05-29T21:48:49.257073Z","shell.execute_reply":"2023-05-29T21:48:49.261286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del embed_vectors,sentence_list,similarity_matrix\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:49:09.945582Z","iopub.execute_input":"2023-05-29T21:49:09.945968Z","iopub.status.idle":"2023-05-29T21:49:10.630194Z","shell.execute_reply.started":"2023-05-29T21:49:09.945939Z","shell.execute_reply":"2023-05-29T21:49:10.628986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Method 2 : Cluster articles using DBSCAN","metadata":{}},{"cell_type":"code","source":"nlp=spacy.load('en_core_web_sm')\nsent_vecs={}\ndocs=[]\n\nfor i in tqdm(all_sources['Title of Competition'].fillna('unknown')[:1000]):\n    doc=nlp((i)) \n    docs.append(doc)\n    sent_vecs.update({i :doc.vector})","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:49:43.293396Z","iopub.execute_input":"2023-05-29T21:49:43.293783Z","iopub.status.idle":"2023-05-29T21:49:51.005023Z","shell.execute_reply.started":"2023-05-29T21:49:43.293754Z","shell.execute_reply":"2023-05-29T21:49:51.003957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences=list(sent_vecs.keys())\nvectors=list(sent_vecs.values())","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:50:22.537831Z","iopub.execute_input":"2023-05-29T21:50:22.538236Z","iopub.status.idle":"2023-05-29T21:50:22.543164Z","shell.execute_reply.started":"2023-05-29T21:50:22.538204Z","shell.execute_reply":"2023-05-29T21:50:22.542305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x=np.array(vectors)\ndbscan=DBSCAN(eps=0.08, min_samples=2,metric='cosine' ).fit(x)\ndf_cluster=pd.DataFrame({'sentences':sentences,'label':dbscan.labels_})","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:50:39.763601Z","iopub.execute_input":"2023-05-29T21:50:39.764039Z","iopub.status.idle":"2023-05-29T21:50:39.774248Z","shell.execute_reply.started":"2023-05-29T21:50:39.764006Z","shell.execute_reply":"2023-05-29T21:50:39.773283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cluster.label.unique()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:50:55.270107Z","iopub.execute_input":"2023-05-29T21:50:55.270931Z","iopub.status.idle":"2023-05-29T21:50:55.278425Z","shell.execute_reply.started":"2023-05-29T21:50:55.270892Z","shell.execute_reply":"2023-05-29T21:50:55.277406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Above: -1 label indicates that the sentence does not belong to any cluster.","metadata":{}},{"cell_type":"markdown","source":"#Some example clusters :","metadata":{}},{"cell_type":"code","source":"df_cluster[(df_cluster['label']==0)].head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:51:38.313358Z","iopub.execute_input":"2023-05-29T21:51:38.313877Z","iopub.status.idle":"2023-05-29T21:51:38.325500Z","shell.execute_reply.started":"2023-05-29T21:51:38.313843Z","shell.execute_reply":"2023-05-29T21:51:38.324442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cluster[(df_cluster['label']==1)].head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T21:52:11.020385Z","iopub.execute_input":"2023-05-29T21:52:11.020796Z","iopub.status.idle":"2023-05-29T21:52:11.032418Z","shell.execute_reply.started":"2023-05-29T21:52:11.020764Z","shell.execute_reply":"2023-05-29T21:52:11.031288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Rake","metadata":{}},{"cell_type":"code","source":"ind,title=list(map(list,zip(*similar)))\ntitles=[]\ntexts=[]\nfor i in ind:\n    titles.append(all_sources.iloc[i]['Title of Competition'])\n    texts.append(all_sources.iloc[i]['Title of Writeup'])","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-29T22:10:00.895141Z","iopub.execute_input":"2023-05-29T22:10:00.895537Z","iopub.status.idle":"2023-05-29T22:10:00.903609Z","shell.execute_reply.started":"2023-05-29T22:10:00.895509Z","shell.execute_reply":"2023-05-29T22:10:00.902373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef clean(txt):\n    txt=re.sub(r'\\n','',txt)\n    txt=re.sub(r'\\([^()]*\\)','',txt)\n    txt=re.sub(r'https?:\\S+\\sdoi','',txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:10:25.172212Z","iopub.execute_input":"2023-05-29T22:10:25.172601Z","iopub.status.idle":"2023-05-29T22:10:25.178530Z","shell.execute_reply.started":"2023-05-29T22:10:25.172573Z","shell.execute_reply":"2023-05-29T22:10:25.177353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts=list(map(clean,texts))\ntext_list=' '.join(texts)\n#text_list=word_tokenize(text_list)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:10:40.367148Z","iopub.execute_input":"2023-05-29T22:10:40.367523Z","iopub.status.idle":"2023-05-29T22:10:40.373393Z","shell.execute_reply.started":"2023-05-29T22:10:40.367495Z","shell.execute_reply":"2023-05-29T22:10:40.372437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Knowledge Graph\n\nNext,we will try and draw a knowledge graph from the related papers we found.","metadata":{}},{"cell_type":"code","source":"import spacy\nnlp=spacy.load('en_core_web_sm')","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-29T22:02:19.884590Z","iopub.execute_input":"2023-05-29T22:02:19.885017Z","iopub.status.idle":"2023-05-29T22:02:21.055074Z","shell.execute_reply.started":"2023-05-29T22:02:19.884985Z","shell.execute_reply":"2023-05-29T22:02:21.054076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_entities(sent):\n    ## chunk 1\n    ent1 = \"\"\n    ent2 = \"\"\n\n    prv_tok_dep = \"\"    # dependency tag of previous token in the sentence\n    prv_tok_text = \"\"   # previous token in the sentence\n\n    prefix = \"\"\n    modifier = \"\"\n\n  #############################################################\n  \n    for tok in nlp(sent):\n        ## chunk 2\n        # if token is a punctuation mark then move on to the next token\n        if tok.dep_ != \"punct\":\n          # check: token is a compound word or not\n          if tok.dep_ == \"compound\":\n            prefix = tok.text\n            # if the previous word was also a 'compound' then add the current word to it\n            if prv_tok_dep == \"compound\":\n                   prefix = prv_tok_text + \" \"+ tok.text\n      \n      # check: token is a modifier or not\n        if tok.dep_.endswith(\"mod\") == True:\n            modifier = tok.text\n            # if the previous word was also a 'compound' then add the current word to it\n            if prv_tok_dep == \"compound\":\n              modifier = prv_tok_text + \" \"+ tok.text\n\n          ## chunk 3\n        if tok.dep_.find(\"subj\") == True:\n            ent1 = modifier +\" \"+ prefix + \" \"+ tok.text\n            prefix = \"\"\n            modifier = \"\"\n            prv_tok_dep = \"\"\n            prv_tok_text = \"\"      \n\n          ## chunk 4\n        if tok.dep_.find(\"obj\") == True:\n            ent2 = modifier +\" \"+ prefix +\" \"+ tok.text\n\n          ## chunk 5  \n          # update variables\n        prv_tok_dep = tok.dep_\n        prv_tok_text = tok.text\n  #############################################################\n\n    return [ent1.strip(), ent2.strip()]","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:03:29.754756Z","iopub.execute_input":"2023-05-29T22:03:29.755163Z","iopub.status.idle":"2023-05-29T22:03:29.766520Z","shell.execute_reply.started":"2023-05-29T22:03:29.755137Z","shell.execute_reply":"2023-05-29T22:03:29.765532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_relation(sent):\n\n  doc = nlp(sent)\n\n  # Matcher class object \n  matcher = Matcher(nlp.vocab)\n\n  #define the pattern \n  pattern = [{'DEP':'ROOT'}, \n            {'DEP':'prep','OP':\"?\"},\n            {'DEP':'agent','OP':\"?\"},  \n            {'POS':'ADJ','OP':\"?\"}] \n\n  matcher.add(\"matching_1\", [pattern]) #add() takes exactly 2 positional arguments (3 given)\n\n  matches = matcher(doc)\n  k = len(matches) - 1\n\n  span = doc[matches[k][1]:matches[k][2]] \n\n  return(span.text)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:24:07.961262Z","iopub.execute_input":"2023-05-29T22:24:07.961724Z","iopub.status.idle":"2023-05-29T22:24:07.969801Z","shell.execute_reply.started":"2023-05-29T22:24:07.961673Z","shell.execute_reply":"2023-05-29T22:24:07.968550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Above Check Spacy Matcher\n\nhttps://spacy.io/api/matcher\n\nadd() takes exactly 2 positional arguments (3 given)","metadata":{}},{"cell_type":"code","source":"def prepare_df(text_list):\n    doc=nlp(text_list)\n    df=pd.DataFrame()\n    for sent in list(doc.sents):\n        sub,obj = get_entities(str(sent))\n        relation= get_relation(str(sent))\n\n        if ((len(relation)>2) & (len(sub)>2) &(len(obj)>2)):\n            df=df.append({'subject':sub,'relation':relation,'object':obj},ignore_index=True)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:24:13.487719Z","iopub.execute_input":"2023-05-29T22:24:13.488842Z","iopub.status.idle":"2023-05-29T22:24:13.495498Z","shell.execute_reply.started":"2023-05-29T22:24:13.488804Z","shell.execute_reply":"2023-05-29T22:24:13.494581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = prepare_df(text_list[24:])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:24:18.341501Z","iopub.execute_input":"2023-05-29T22:24:18.341890Z","iopub.status.idle":"2023-05-29T22:24:18.383619Z","shell.execute_reply.started":"2023-05-29T22:24:18.341859Z","shell.execute_reply":"2023-05-29T22:24:18.382904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_kg(pairs,c1='red',c2='blue',c3='orange'):\n    k_graph = nx.from_pandas_edgelist(pairs, 'subject', 'object',\n            create_using=nx.MultiDiGraph())\n  \n    node_deg = nx.degree(k_graph)\n    layout = nx.spring_layout(k_graph, k=0.15, iterations=20)\n    plt.figure(num=None, figsize=(50, 40), dpi=80)\n    nx.draw_networkx(\n        k_graph,\n        node_size=[int(deg[1]) * 500 for deg in node_deg],\n        arrowsize=20,\n        linewidths=1.5,\n        pos=layout,\n        edge_color=c1,\n        edgecolors=c2,\n        node_color=c3,\n        )\n    labels = dict(zip(list(zip(pairs.subject, pairs.object)),\n                  pairs['relation'].tolist()))\n    nx.draw_networkx_edge_labels(k_graph, pos=layout, edge_labels=labels,\n                                 font_color='red')\n    plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:24:48.277568Z","iopub.execute_input":"2023-05-29T22:24:48.277945Z","iopub.status.idle":"2023-05-29T22:24:48.287841Z","shell.execute_reply.started":"2023-05-29T22:24:48.277917Z","shell.execute_reply":"2023-05-29T22:24:48.286739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#I connected 2 dots, though I couldn't read them.","metadata":{}},{"cell_type":"code","source":"draw_kg(df)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:25:05.270340Z","iopub.execute_input":"2023-05-29T22:25:05.270727Z","iopub.status.idle":"2023-05-29T22:25:06.294346Z","shell.execute_reply.started":"2023-05-29T22:25:05.270669Z","shell.execute_reply":"2023-05-29T22:25:06.293299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#I can't even read what's written on the extremities.","metadata":{}},{"cell_type":"code","source":"tasks=[\"Santa\",\n      \"Bowl\",\n      \"RSNA\",\n      \"toxic\",\n      \"fraud\",\n      \"Segmentation\",\n      \"prize\",\n      \"challenge\",\n      \"detection\",\n      ]","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:31:28.830246Z","iopub.execute_input":"2023-05-29T22:31:28.830712Z","iopub.status.idle":"2023-05-29T22:31:28.836467Z","shell.execute_reply.started":"2023-05-29T22:31:28.830654Z","shell.execute_reply":"2023-05-29T22:31:28.835259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Sentence came from input 46","metadata":{}},{"cell_type":"code","source":"sentence= \"RTA Freeway Travel Time Prediction\"\nsimilar=get_top_similar(sentence,sentence_list, similarity_matrix,15)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:38:34.078207Z","iopub.execute_input":"2023-05-29T22:38:34.078609Z","iopub.status.idle":"2023-05-29T22:38:34.083956Z","shell.execute_reply.started":"2023-05-29T22:38:34.078581Z","shell.execute_reply":"2023-05-29T22:38:34.082750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ind,title=list(map(list,zip(*similar)))\ntitles=[]\ntexts=[]\nfor i in ind:\n    titles.append(all_sources.iloc[i]['Title of Competition'])\n    texts.append(all_sources.iloc[i]['Title of Writeup'])","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:40:29.433841Z","iopub.execute_input":"2023-05-29T22:40:29.434247Z","iopub.status.idle":"2023-05-29T22:40:29.442023Z","shell.execute_reply.started":"2023-05-29T22:40:29.434217Z","shell.execute_reply":"2023-05-29T22:40:29.441164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts=list(map(clean,texts))\ntext_list=' '.join(texts)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:40:52.150748Z","iopub.execute_input":"2023-05-29T22:40:52.151137Z","iopub.status.idle":"2023-05-29T22:40:52.156645Z","shell.execute_reply.started":"2023-05-29T22:40:52.151107Z","shell.execute_reply":"2023-05-29T22:40:52.155418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#That draw is almost a meme. Therefore, No final funny images.","metadata":{}},{"cell_type":"code","source":"df = prepare_df(text_list)\ndraw_kg(df)","metadata":{"execution":{"iopub.status.busy":"2023-05-29T22:41:08.905284Z","iopub.execute_input":"2023-05-29T22:41:08.906017Z","iopub.status.idle":"2023-05-29T22:41:09.978744Z","shell.execute_reply.started":"2023-05-29T22:41:08.905985Z","shell.execute_reply":"2023-05-29T22:41:09.977624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#The shortest distance between two points is always a straight line, but only in Euclidean geometry.\n\nAfter that, I gave up since there is private source and I couldn't find all the snippets.\n\nCodes found on Version 26.","metadata":{}},{"cell_type":"markdown","source":"#Acknowledgements:\n\nShahules https://www.kaggle.com/code/shahules/cord-tools-and-knowledge-graphs/notebook\n\nBrian universal-sentence-encoder-large-4\nhttps://www.kaggle.com/datasets/ldm314/universalsentenceencoderlarge4\n\nMatthew Honnibal spacy-en_vectors_web_lg https://www.kaggle.com/datasets/honnibal/spacyen-vectors-web-lg","metadata":{}}]}