{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this Notebook\nThis is a first run through the compeition to try and understand the datatset and realise the problem at hand.","metadata":{}},{"cell_type":"code","source":"# Asthetics\nimport warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\n# Basic\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nimport json\nimport os\nimport random\nfrom tqdm.autonotebook import tqdm\nimport string\nimport re\nfrom functools import partial\n\n# Visualizations\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\nfrom wordcloud import WordCloud, STOPWORDS\n\n# NLP\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 40000000","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Description\n\ntrain.csv -labels and metadata for the training set\ntrain/tezt directory - the full text of the training/test set's publications in JSON format, broken into sections with section titles\n* `id` - publication id - note that there are multiple rows for some training documents, indicating multiple mentioned datasets.\n* `pub_title` - title of the publication (a small number of publications have the same title).\n* `dataset_title` - the title of the dataset that is mentioned within the publication.\n* `dataset_label` - a portion of the text that indicates the dataset.\n* `cleaned_label` - the dataset_label, as passed through the clean_text function from the [Evaluation page](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation).\n\nsample_submission.csv - a sample submission file in the correct format.\n* `Id` - publication id.\n* `PredictionString` - To be filled with equivalent of `cleaned_label` of train data.","metadata":{}},{"cell_type":"code","source":"RANDOM_SEED = 42","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=RANDOM_SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's have a look at the training data csv file...","metadata":{}},{"cell_type":"code","source":"train_df.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's get the text data from json files and append them to the table.","metadata":{}},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's save the data now in case we needed that while creating model later.","metadata":{}},{"cell_type":"code","source":"# train_df.to_csv('Data_with_text.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Cleaning","metadata":{}},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain_df['text'] = train_df['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tqdm.pandas()\n# sample_sub['text'] = sample_sub['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generating Word cloud","metadata":{}},{"cell_type":"code","source":"text = ' '.join(train_df['text'].sample(frac=0.3))\nwordcloud = WordCloud(background_color='white', stopwords=STOPWORDS, width=2560, height=1440).generate(text)\n\nbarplot_dim = (15, 15)\nax = plt.subplots(figsize=barplot_dim, facecolor='w')\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see there are metions of 'et al' pretty significantly in the test of papers. Which is in fact related to quoting papers. This this should be a significant factor in determining the citation titles. Let, hope so...","metadata":{}},{"cell_type":"markdown","source":"# Preparing text","metadata":{}},{"cell_type":"code","source":"def prepare_text(text, nlp=nlp):\n    '''\n    Returns the text after stop-word removal and lemmatization.\n    text - Sentence to be processed\n    nlp - Spacy NLP model\n    '''\n    doc = nlp(text)\n    lemma_list = [token.lemma_ for token in doc if not token.is_stop]\n    lemmatized_sentence = ' '.join(lemma_list)\n    \n    return lemmatized_sentence","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tqdm.pandas()\n# train_df['text'] = train_df['text'].progress_apply(prepare_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\nThis is a very naive model based on the assumption that topics having names of label or dataset_title in their content most porobaby are citing the same sources.","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\n'''\nIdea below of also using the 'dataset_title' is burrowed from\nhttps://www.kaggle.com/josephassaker/coleridge-initiative-eda-na-ve-submission\n'''\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = train_df[train_df['text'] == text_cleaning(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = id_list\nsubmission['PredictionString'] = lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This notebook is a work in progress... This is just a first pass through the data to see what is the situation, along with a very Naive model. Even with that, trust me this used to have LB = 1.0 at some point in time. 😆😛  \n\n**If you found this notebook useful and use parts of it in your work, please don't forget to show your appreciation by upvoting this kernel. That keeps me motivated and inspires me to write and share these public kernels.** 😊","metadata":{}}]}