{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n#import numpy as np # linear algebra\n#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n       # print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-29T15:27:02.607401Z","iopub.execute_input":"2021-06-29T15:27:02.607804Z","iopub.status.idle":"2021-06-29T15:27:02.612080Z","shell.execute_reply.started":"2021-06-29T15:27:02.607724Z","shell.execute_reply":"2021-06-29T15:27:02.611388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#2Approach to detect datasources: Countvectorizer with n-grams and \nimport re,nltk,gensim\nimport json as js\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom nltk.corpus import stopwords\nfrom gensim.models.ldamodel import LdaModel\n\ndef get_data_sources(tex):\n    f_hand=open(tex,'r')\n    text=js.load(f_hand)\n    #convert json to string\n    full_text=''\n    for i in range(0,len(text)):\n        full_text=full_text+' '+text[i]['text']\n    #free from punctuation pharantheses and dates    \n    # FIND CAPITAL LETTERS\n    capital_letters=re.findall(r'([A-Z]{3,})',full_text)\n    capitalletters=list(capital_letters)\n    dict_capitalletters={}\n    for i in capitalletters:\n        dict_capitalletters[i]=dict_capitalletters.get(i,0)+1\n    #dict_capitalletters\n    #REPLACE CAPICAL LETTERS with long versions\n    #here we had to remove stop words in order to find out long versions of datasources in text\n    ffs=[w for w in nltk.word_tokenize(full_text) if w.lower() not in stopwords.words('english')]\n    ffs_text=' '.join(ffs)\n    for i in ffs:\n        for cap_let in set(capitalletters):\n            if i==cap_let:\n                word_to_find=[cl+'[A-Za-z]+' for cl in cap_let]\n                wrdtofind=' '.join(word_to_find)\n                #print(wrdtofind)\n                #print(re.search(wrdtofind,ffs_text))\n                try:\n                    ffs_text=re.sub(i,re.search(wrdtofind ,ffs_text)[0],ffs_text)\n                    #ffs_text=re.sub(i,re.search(wrdtofind.lower(),ffs_text)[0],ffs_text)\n                except:\n                    continue\n            else:\n                continue\n\n    #free from punctuation and pharantheses\n    free_from_symb_dates=[]\n    for i in nltk.sent_tokenize(ffs_text):\n        free_from_symb_dates.append(re.sub(r'\\b\\w\\w\\b','',re.sub(r'\\d+','',re.sub(r'\\W+',' ', i)))+'.')\n\n    #data Sentences \n    data_words=['Study','Survey','Statistics','Data','Dataset','Database','Census','Assessment']\n\n    data_sentences=[w for w in free_from_symb_dates for i in range(0,len(data_words)) if data_words[i].lower() in w.lower()]\n    #countvectorizer and fit and transform text \n    vectorizer=CountVectorizer(analyzer='word',max_df=0.7,ngram_range=(3,5))# ngram_range=(3,5) ,\n    fitted_model=vectorizer.fit_transform(set(data_sentences))\n    # create corpus to use in Lda model\n    corpus = gensim.matutils.Sparse2Corpus(fitted_model, documents_columns=False)\n    id_map = dict((v, k) for k, v in vectorizer.vocabulary_.items())\n\n    # Lda model\n    ldamodel=LdaModel(corpus,num_topics=1,id2word=id_map,per_word_topics=True,passes=50)\n    return ldamodel.show_topic(topicid=0,topn=4)[0][0]+'|'+ldamodel.show_topic(topicid=0,topn=4)[1][0]+'|'+ldamodel.show_topic(topicid=0,topn=4)[2][0]\n\nheader='Id,PredictionString'\nfile_open=open('submission.csv','w')\nfile_open.write(header+\"\\n\")\nfile_open.close()\ntext_ids=['2100032a-7c33-4bff-97ef-690822c43466',\n           '2f392438-e215-4169-bebf-21ac4ff253e1',\n           '3f316b38-1a24-45a9-8d8c-4e05a42257c6',\n           '8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60']\n    \nwith open('submission.csv', 'a') as f:\n    for ids in text_ids:\n        tex='../input/coleridgeinitiative-show-us-the-data/test/'+ids+'.json'\n        f.write(ids+ ','+get_data_sources(tex)+\"\\n\")\nf.close()","metadata":{"execution":{"iopub.status.busy":"2021-06-29T15:28:10.361591Z","iopub.execute_input":"2021-06-29T15:28:10.361957Z","iopub.status.idle":"2021-06-29T15:28:41.515372Z","shell.execute_reply.started":"2021-06-29T15:28:10.361929Z","shell.execute_reply":"2021-06-29T15:28:41.514530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}