{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nimport nltk\nimport spacy\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os\nfrom tqdm import tqdm\nfrom functools import partial\nimport re\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom nltk.corpus import stopwords\nfrom spacy.lang.en.stop_words import STOP_WORDS\nimport itertools\nimport collections\nfrom nltk.stem import WordNetLemmatizer\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.metrics.pairwise import cosine_similarity, euclidean_distances\nfrom sklearn.feature_extraction.text import TfidfVectorizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Define Paths for Train and Test Json files\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Function to read JSON files and extract publication Text \n\ndef json_to_text(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n            \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = ' '.join(combined)\n    \n    if output=='text':\n        return all_contents\n    elif output=='head':\n        return all_headings\n    else:\n        return all_data","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Extract Publication Text for Training Data\ntqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(json_to_text)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Reading the Sample Submission Data\n\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsample_sub.head()\n### Extract Publication Text for the sample publications \nsample_sub['text'] = sample_sub['Id'].apply(partial(json_to_text,train_files_path=test_files_path))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lemmatization(text):\n\n    doc = nlp(text)\n    lemma_list = [token.lemma_ for token in doc if not token.is_stop]\n    return ' '.join(lemma_list)\n\ndef nltk_lemma(text):\n    lemmatizer = WordNetLemmatizer()\n    words = text.split()\n    return [lemmatizer.lemmatize(word) for word in words]\n    #return lemmatizer.lemmatize(text)\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text'] = train_df['text'].progress_apply(clean_text)\n\nstop_words = stopwords.words('english')\ntrain_df['text'] = train_df['text'].progress_apply(nltk_lemma)\n\ntrain_df['text'] = train_df['text'].apply(\" \".join)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docs = train_df['text'].tolist()\n\n#Ignore words that appear in 85% texts, \ncv = CountVectorizer(max_df=0.85, stop_words=stop_words, max_features=60000)\nword_count_vector = cv.fit_transform(docs)\n\ntfidf_transformer = TfidfTransformer(smooth_idf=True,use_idf=True)\ntfidf_transformer.fit(word_count_vector)\nfeature_names = cv.get_feature_names()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sort_coo(coo_matrix):\n    tuples = zip(coo_matrix.col, coo_matrix.data)\n    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)\n\ndef extract_topn_from_vector(feature_names, sorted_items, topn=10):\n    \"\"\"get the feature names and tf-idf score of top n items\"\"\"\n    \n    #use only topn items from vector\n    sorted_items = sorted_items[:topn]\n\n    score_vals = []\n    feature_vals = []\n    \n    # word index and corresponding tf-idf score\n    for idx, score in sorted_items:\n        \n        #keep track of feature name and its corresponding score\n        score_vals.append(round(score, 3))\n        feature_vals.append(feature_names[idx])\n\n    #create a tuples of feature,score\n    results= {}\n    for idx in range(len(feature_vals)):\n        results[feature_vals[idx]]=score_vals[idx]\n    \n    return results","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Ids = train_df.Id.tolist()\nkeyword_df = pd.DataFrame()\n\nfor i in range(len(docs)):\n    doc = docs[i]\n    Id = Ids[i]\n    tfidf_vector = tfidf_transformer.transform(cv.transform([doc]))\n\n    #sort the tf-idf vectors by descending order of scores\n    sorted_items=sort_coo(tfidf_vector.tocoo())\n\n    #extract only the top n; n here is 500\n    keywords=extract_topn_from_vector(feature_names,sorted_items,500)\n\n    temp_df = pd.DataFrame()\n    temp_df['keyword'] = keywords\n    temp_df['weight'] = keywords.values()\n    temp_df['id']=Id\n    keyword_df = keyword_df.append(temp_df)\n\n### We have a DataFrame with Keywords for each article and its keywords with their weights\nkeyword_df[['id','keyword','weight']].head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_df.to_csv('keywords_df.csv')\n#keyword_df=pd.read_csv('../input/end-2-end-cosine-similarity/keywords_df.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_grpd = keyword_df.groupby('id')['keyword'].unique().reset_index()\ndf_train_key = pd.merge(train_df,keyword_grpd, left_on='Id', right_on='id')\n\ndf_train_key_grpd = df_train_key.copy()\ndf_train_key_grpd['keyword'] = [str(x) for x in df_train_key_grpd['keyword']]\ndf_train_key_grpd = df_train_key_grpd.groupby('cleaned_label')['keyword'].sum().reset_index()\n\ndf_train_key_grpd.loc[:,'keyword'] = df_train_key_grpd.keyword.str.replace('[','')\ndf_train_key_grpd.loc[:,'keyword'] = df_train_key_grpd.keyword.str.replace(']','')\ndf_train_key_grpd.loc[:,'keyword'] = df_train_key_grpd.keyword.str.replace('\\n','')\ndf_train_key_grpd.loc[:,'keyword'] = df_train_key_grpd['keyword'].str.split(\" \").map(set).str.join(\" \")\n\ndf_train_key_grpd.loc[:,'keyword'] = df_train_key_grpd['keyword'].str.replace(\"''\",\" \")\ndf_train_key_grpd.loc[:,'keyword'] = df_train_key_grpd['keyword'].str.replace(\"'\",\"\") \n\ndf_train_key_grpd.loc[:,'keyword'] = df_train_key_grpd['keyword'].str.split(\" \").map(set).str.join(\" \")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_key_grpd.to_csv('df_train_key_grpd.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_train_key_grpd.rename(columns={'Unnamed: 0':'ix'}, inplace=True)\ndf_train_key_grpd.reset_index(inplace=True)\n\ntqdm.pandas()\nsample_sub['text'] = sample_sub['text'].progress_apply(clean_text)\nsample_sub['text'] = sample_sub['text'].progress_apply(nltk_lemma)\n\nsample_sub['text'] = sample_sub['text'].apply(\" \".join)\n\ndocs = sample_sub['text'].tolist()\n\n\nIds = sample_sub.Id.tolist()\nkeyword_df = pd.DataFrame()\n\nfor i in range(len(docs)):\n    doc = docs[i]\n    Id = Ids[i]\n    tfidf_vector = tfidf_transformer.transform(cv.transform([doc]))\n\n    sorted_items=sort_coo(tfidf_vector.tocoo())\n\n    #extract only the top n; n here is 500\n    keywords=extract_topn_from_vector(feature_names,sorted_items,500)\n\n    temp_df = pd.DataFrame()\n    temp_df['keyword'] = keywords\n    temp_df['weight'] = keywords.values()\n    temp_df['id']=Id\n    keyword_df = keyword_df.append(temp_df)\n\nkeyword_df[['id','keyword','weight']].head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_grpd_test = keyword_df.groupby('id')['keyword'].unique().reset_index()\ndf_test_key = pd.merge(sample_sub,keyword_grpd_test, left_on='Id', right_on='id')\n\ndf_test_key_grpd = df_test_key.copy()\ndf_test_key_grpd['keyword'] = [str(x) for x in df_test_key_grpd['keyword']]\ndf_test_key_grpd = df_test_key_grpd.groupby('Id')['keyword'].sum().reset_index()\n\ndf_test_key_grpd.loc[:,'keyword'] = df_test_key_grpd.keyword.str.replace('[','')\ndf_test_key_grpd.loc[:,'keyword'] = df_test_key_grpd.keyword.str.replace(']','')\ndf_test_key_grpd.loc[:,'keyword'] = df_test_key_grpd.keyword.str.replace('\\n','')\ndf_test_key_grpd.loc[:,'keyword'] = df_test_key_grpd['keyword'].str.split(\" \").map(set).str.join(\" \")\n\ndf_test_key_grpd.loc[:,'keyword'] = df_test_key_grpd['keyword'].str.replace(\"''\",\" \")\ndf_test_key_grpd.loc[:,'keyword'] = df_test_key_grpd['keyword'].str.replace(\"'\",\"\") \n\ndf_test_key_grpd.loc[:,'keyword'] = df_test_key_grpd['keyword'].str.split(\" \").map(set).str.join(\" \")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfidfvectoriser = TfidfVectorizer()\ntfidfvectoriser.fit(df_train_key_grpd.keyword)\ndataset_tfidf_vectors = tfidfvectoriser.transform(df_train_key_grpd.keyword)\n\ntest_tfidf_vectors = tfidfvectoriser.transform(df_test_key_grpd.keyword)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pairwise_similarities = np.dot(test_tfidf_vectors, dataset_tfidf_vectors.T).toarray()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### - take top 3 matching dataset\nmatching_df = pd.DataFrame()\n\nfor i,val in enumerate(df_test_key_grpd.Id):\n    temp_df = pd.DataFrame()\n    sim_index = np.argsort(pairwise_similarities[i])[:126:-1]\n    temp_df['similar_dataset'] = sim_index\n    temp_df['similarity_score'] = pairwise_similarities[i][sim_index]\n    temp_df['Id'] = val\n    temp_df['ix'] = i\n    matching_df = matching_df.append(temp_df)\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matched_df_merge = pd.merge(matching_df[['similar_dataset','Id']],\n                            df_train_key_grpd[['index','cleaned_label']], left_on='similar_dataset',right_on='index',\n                            how='inner')\n\nsub_df = matched_df_merge.groupby('Id')['cleaned_label'].apply('|'.join).reset_index()\nsub_df.rename(columns={'cleaned_label':'PredictionString'}, inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filter_matching_df = matching_df[matching_df.similarity_score>=0.08].copy()\n#matched_df_merge = pd.merge(filter_matching_df[['similar_dataset','Id','ix']],\n#                            df_train_key_grpd[['index','cleaned_label']], left_on='similar_dataset',right_on='index',\n#                            how='inner')\n#\n#sub_df = matched_df_merge.groupby('Id')['cleaned_label'].apply('|'.join).reset_index()\n#sub_df_merge = pd.merge(df_test_key_grpd,sub_df, on='Id', how='left')\n#sub_df_merge['cleaned_label'].fillna('alzheimer s disease neuroimaging initiative adni|adni', inplace=True)\n#sub_df_merge = sub_df_merge[['Id','cleaned_label']]\n#sub_df_merge.rename(columns={'cleaned_label':'PredictionString'}, inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index = False)\n#sub_df_merge.to_csv('submission.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]}]}