{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\nI added a *Naive CV* (**Local Validation**) so that you can track your improvement.\n\n## References\nPlease check them out.\n- [@prashansdixit](https://www.kaggle.com/prashansdixit)\n - [📝Coleridge Initiative-EDA📚 & Baseline Model🎯](https://www.kaggle.com/prashansdixit/coleridge-initiative-eda-baseline-model)\n- [@mghfarahani](https://www.kaggle.com/mghfarahani)\n - [Coleridge Initiative - Analysis](https://www.kaggle.com/mghfarahani/coleridge-initiative-analysis)\n- [@mlconsult](https://www.kaggle.com/mlconsult)\n - [score 57ish with additional govt datasets](https://www.kaggle.com/mlconsult/score-57ish-with-additional-govt-datasets)\n \n## Process\n- What\n - The objective of the competition is to identify the mention of datasets within scientific publications.\n- How\n - By literally extracting context and compare with labels we collected. (Baseline)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Setting","metadata":{}},{"cell_type":"markdown","source":"## Libraries","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\nimport os, re, json, glob\nfrom collections import defaultdict\nfrom textblob import TextBlob\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport seaborn as sns\n\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4_000_000\nimport nltk\nfrom nltk.probability import FreqDist\nfrom nltk.corpus import stopwords\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom tqdm.autonotebook import tqdm\nimport string\n\n%matplotlib inline\n\nos.listdir('../input/coleridgeinitiative-show-us-the-data')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"COMPUTE_CV = True\nKEN_TEXT_CLEANING = False\nNLTK_STOPWORDS = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\nif len(sample_sub) > 4: COMPUTE_CV = False\n    \nif COMPUTE_CV: \n    print('this submission notebook will compute CV score but commit notebook will not')\nelse:\n    print('this submission notebook will only be used to submit result')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\n\nif COMPUTE_CV: \n    sample_sub = train_df\nelse:\n    sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n    test_files_path = '../input/coleridgeinitiative-show-us-the-data/test'\n\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Description**\n- `id` - publication id - note that there are multiple rows for some training documents, indicating multiple mentioned datasets.\n- `pub_title` - title of the publication (a small number of publications have the same title).\n- `dataset_title` - the title of the dataset that is mentioned within the publication.\n- `dataset_label` - a portion of the text that indicates the dataset.\n- `cleaned_label` - the dataset_label, as passed through the clean_text function from the Evaluation page.","metadata":{}},{"cell_type":"code","source":"sample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**sample_submission.csv** - a sample submission file in the correct format.\n- `Id` - publication id.\n- `PredictionString` - To be filled with equivalent of cleaned_label of train data.","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unique Values","metadata":{}},{"cell_type":"code","source":"[print(f'{col}: {len( train_df[col].unique() )}') for col in train_df.columns]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Contents","metadata":{}},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\n\ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)\n\nif not COMPUTE_CV:\n    sample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return,\n                                                             train_files_path=test_files_path))\n\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Text Cleaning","metadata":{}},{"cell_type":"code","source":"if KEN_TEXT_CLEANING:\n    \n    # from https://www.kaggle.com/mlconsult/score-57ish-with-additional-govt-datasets\n    def text_cleaning(text):\n        '''\n        Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n        text - Sentence that needs to be cleaned\n        '''\n        text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n        text = re.sub(' +', ' ', text)\n        emoji_pattern = re.compile(\"[\"\n                                   u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                                   u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                                   u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                                   u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                                   \"]+\", flags=re.UNICODE)\n        text = emoji_pattern.sub(r'', text)\n        return text\n    \nelse:\n    \n    def text_cleaning(text):\n        '''\n        Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n        text - Sentence that needs to be cleaned\n        '''\n        text = ''.join([k for k in text if k not in string.punctuation])\n        text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n        # text = re.sub(\"/'+/g\", ' ', text)\n        return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\n\ntrain_df['text'] = train_df['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Vizualization","metadata":{}},{"cell_type":"code","source":"words = list( train_df['cleaned_label'].values )\n\nif NLTK_STOPWORDS:\n    stopwords = stopwords.words('english')\nelse:\n    stopwords = ['ourselves', 'hers', 'the', 'of', 'and', 'in', 'between', 'yourself', 'but', 'again', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than', '2', '19', 'dataset', 'c', 'database']\n\nsplit_words = []\nfor word in words:\n    lo_w = []\n    list_of_words = str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\n    \nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 100 Most Common Words\n`cleaned_label` - WordCloud","metadata":{}},{"cell_type":"code","source":"mostcommon = FreqDist(allwords).most_common(100)\nwordcloud = WordCloud( width = 1600,\n                      height = 800,\n                      background_color = 'white',\n                      stopwords = STOPWORDS ).generate(str(mostcommon))\nfig = plt.figure(figsize=(30, 10), facecolor='white')\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis('off')\nplt.title('Top 100 Most Common Words in cleaned_label', fontsize=50)\nplt.tight_layout(pad=0)\nplt.show()\n\nmostcommon_small = FreqDist(allwords).most_common(25)\nx, y = zip(*mostcommon_small)\nplt.figure(figsize=(50, 30))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('words', fontsize=50)\nplt.ylabel('Frequency of Words', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40)\nplt.title('Freq of 25 Most Common Words in cleaned_label', fontsize=60)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline model and Submission","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\n\nprint(f'len(temp_1) = {len(temp_1)}')\nprint(f'len(temp_2) = {len(temp_2)}')\nprint(f'len(temp_3) = {len(temp_3)}')\nprint(f'len(existing_labels) = {len(existing_labels)}')\n\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = train_df[train_df['text'] == text_cleaning(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    \n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n            \n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['Id'] = id_list\nsample_sub['PredictionString'] = lables_list\nsample_sub[['Id', 'PredictionString']].to_csv('submission.csv', index=False)\n\nsample_sub[['Id', 'PredictionString']].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute CV","metadata":{}},{"cell_type":"code","source":"def getMetric(col):\n    def f1score(row):\n        n = len(np.intersect1d(row.PredictionString.split('|'), row[col]))\n        return 2*n / (len(row.PredictionString.split('|')) + len(row[col]))\n    return f1score\n\ndef my_jaccard(strs): \n    str1, str2 = strs\n    temp_list = []\n    for sentence in str1.lower().split('|'):\n        a = set(str1.lower().split()) \n        b = set(str2.lower().split())\n        c = a.intersection(b)\n        d = float(len(c)) / (len(a) + len(b) - len(c))\n        temp_list.append(d)\n    return sum(temp_list) / len(temp_list)   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COMPUTE_CV:\n    getMetric_score = sample_sub.apply(getMetric('cleaned_label'), axis=1)\n    print('getMetric_score =', getMetric_score.mean())\n    my_jaccard_score = sample_sub[['PredictionString', 'cleaned_label']].apply(my_jaccard, axis=1)\n    print('my_jaccard_score =', my_jaccard_score.mean())\n    \nprint(f'COMPUTE_CV = {COMPUTE_CV}')\nprint(f'KEN_TEXT_CLEANING = {KEN_TEXT_CLEANING}')\nprint(f'NLTK_STOPWORDS = {NLTK_STOPWORDS}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"|   | CV | LB |\n| --- | --- | --- |\n| KEN + MY_SW | 0.705 | 0.534 |\n| MY_SW | 0.695 | 0.534 |\n| NLTK_SW | 0.700 | 0.534 |\n| KEN + NLTK_SW | 0.701 | 0.534 |","metadata":{}}]}