{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport json\nimport re\nimport glob\nimport spacy\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom nltk.tag.sequential import ClassifierBasedPOSTagger","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-20T18:54:30.836974Z","iopub.execute_input":"2021-06-20T18:54:30.837513Z","iopub.status.idle":"2021-06-20T18:54:32.925976Z","shell.execute_reply.started":"2021-06-20T18:54:30.837432Z","shell.execute_reply":"2021-06-20T18:54:32.925069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load train.csv\ntrain_csv = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:32.927397Z","iopub.execute_input":"2021-06-20T18:54:32.927648Z","iopub.status.idle":"2021-06-20T18:54:33.086798Z","shell.execute_reply.started":"2021-06-20T18:54:32.927624Z","shell.execute_reply":"2021-06-20T18:54:33.085922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:33.088656Z","iopub.execute_input":"2021-06-20T18:54:33.088971Z","iopub.status.idle":"2021-06-20T18:54:33.09448Z","shell.execute_reply.started":"2021-06-20T18:54:33.08893Z","shell.execute_reply":"2021-06-20T18:54:33.093548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list of training publications\ntrain_pubs = glob.glob(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train/*.json\")","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:33.096086Z","iopub.execute_input":"2021-06-20T18:54:33.096391Z","iopub.status.idle":"2021-06-20T18:54:33.284155Z","shell.execute_reply.started":"2021-06-20T18:54:33.096363Z","shell.execute_reply":"2021-06-20T18:54:33.283188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_sentence(txt):\n     #return re.sub('[^A-Za-z0-9.]+', ' ', str(txt).lower())   \n    return re.sub('[^A-Za-z0-9.]+', ' ', str(txt))   ","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:33.285329Z","iopub.execute_input":"2021-06-20T18:54:33.285623Z","iopub.status.idle":"2021-06-20T18:54:33.29018Z","shell.execute_reply.started":"2021-06-20T18:54:33.285597Z","shell.execute_reply":"2021-06-20T18:54:33.289072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:33.291282Z","iopub.execute_input":"2021-06-20T18:54:33.291551Z","iopub.status.idle":"2021-06-20T18:54:33.31061Z","shell.execute_reply.started":"2021-06-20T18:54:33.291524Z","shell.execute_reply":"2021-06-20T18:54:33.309702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\n\nDATA = []\nfor idx,row in tqdm(train_csv[0:500].iterrows()):\n    TRAIN_DATA=[]\n\n    pub = \"../input/coleridgeinitiative-show-us-the-data/train/\" + row.Id + \".json\"            \n    f = open(pub)  \n    data = json.load(f)      \n\n    sentences = nltk.tokenize.sent_tokenize(str(data))\n    for sentence in sentences:          \n        sentence = clean_sentence(sentence).strip()        \n        \n        #loc = re.search(row.dataset_label.lower(),sentence)\n        loc = re.search(row.dataset_label,sentence)\n        \n        if loc!=None:\n            begin=loc.span()[0]\n            end=loc.span()[1]\n            \n            tokens1 = nltk.word_tokenize(sentence[0:begin])\n            tokens2 = nltk.word_tokenize(sentence[begin:end+1])\n            tokens3 = nltk.word_tokenize(sentence[end+1:])\n            \n            pos_tag1 = nltk.pos_tag(tokens1)\n            pos_tag2 = nltk.pos_tag(tokens2)\n            pos_tag3 = nltk.pos_tag(tokens3)\n            \n            first = True\n            \n            for pos in pos_tag1:\n                TRAIN_DATA.append((pos,\"O\"))\n            for pos in pos_tag2:\n                if first:\n                    TRAIN_DATA.append((pos,\"B-WORK_OF_ART\"))\n                    first = False\n                else:\n                    TRAIN_DATA.append((pos,\"I-WORK_OF_ART\"))\n            for pos in pos_tag3:\n                TRAIN_DATA.append((pos,\"O\"))\n    #print(TRAIN_DATA)\n    DATA.append(TRAIN_DATA)\n            ","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:33.311743Z","iopub.execute_input":"2021-06-20T18:54:33.312047Z","iopub.status.idle":"2021-06-20T18:54:50.846184Z","shell.execute_reply.started":"2021-06-20T18:54:33.312019Z","shell.execute_reply":"2021-06-20T18:54:50.845173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport nltk\nimport string\n\nfrom nltk import pos_tag\nfrom nltk import word_tokenize\nfrom nltk.chunk import ChunkParserI\nfrom nltk.chunk import conlltags2tree, tree2conlltags\nfrom nltk.tag import ClassifierBasedTagger\nfrom nltk.tag.util import untag\nfrom nltk.stem.snowball import SnowballStemmer\n","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:50.847632Z","iopub.execute_input":"2021-06-20T18:54:50.847966Z","iopub.status.idle":"2021-06-20T18:54:50.853494Z","shell.execute_reply.started":"2021-06-20T18:54:50.847938Z","shell.execute_reply":"2021-06-20T18:54:50.852519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IOB tag name for specifying dataset label \nGPE_TAG = \"WORK_OF_ART\"\n\nclass DatasetChunker(ChunkParserI):\n    def __init__(self, train_sents, **kwargs):\n        #print(train_sents)\n        self.tagger = ClassifierBasedTagger(\n            train=train_sents,\n            feature_detector=self.features,\n            **kwargs)\n\n    def parse(self, tagged_sent):\n        chunks = self.tagger.tag(tagged_sent)\n\n        # Transform the result from [((w1, t1), iob1), ...]\n        # to the preferred list of triplets format [(w1, t1, iob1), ...]\n        iob_triplets = [(w, t, c) for ((w, t), c) in chunks]\n\n        # Transform the list of triplets to nltk.Tree format\n        return conlltags2tree(iob_triplets)\n    \n    def features(self, tokens, index, history):\n        # for more details see: http://nlpforhackers.io/named-entity-extraction/ \n        \n        \"\"\"\n        `tokens`  = a POS-tagged sentence [(w1, t1), ...]\n        `index`   = the index of the token we want to extract features for\n        `history` = the previous predicted IOB tags\n        \"\"\"\n\n        # init the stemmer\n        stemmer = SnowballStemmer('english')\n\n        # Pad the sequence with placeholders\n        tokens = [('[START2]', '[START2]'), ('[START1]', '[START1]')] + list(tokens) + [('[END1]', '[END1]'), ('[END2]', '[END2]')]\n        history = ['[START2]', '[START1]'] + list(history)\n\n        # shift the index with 2, to accommodate the padding\n        index += 2\n\n        word, pos = tokens[index]\n        prevword, prevpos = tokens[index - 1]\n        prevprevword, prevprevpos = tokens[index - 2]\n        nextword, nextpos = tokens[index + 1]\n        nextnextword, nextnextpos = tokens[index + 2]\n        previob = history[index - 1]\n        contains_dash = '-' in word\n        contains_dot = '.' in word\n        allascii = all([True for c in word if c in string.ascii_lowercase])\n\n        allcaps = word == word.capitalize()\n        capitalized = word[0] in string.ascii_uppercase\n\n        prevallcaps = prevword == prevword.capitalize()\n        prevcapitalized = prevword[0] in string.ascii_uppercase\n\n        nextallcaps = prevword == prevword.capitalize()\n        nextcapitalized = prevword[0] in string.ascii_uppercase\n\n        f = {\n            'word': word,\n            'lemma': stemmer.stem(word),\n            'pos': pos,\n            'all-ascii': allascii,\n\n            'next-word': nextword,\n            'next-lemma': stemmer.stem(nextword),\n            'next-pos': nextpos,\n\n            'next-next-word': nextnextword,\n            'nextnextpos': nextnextpos,\n\n            'prev-word': prevword,\n            'prev-lemma': stemmer.stem(prevword),\n            'prev-pos': prevpos,\n\n            'prev-prev-word': prevprevword,\n            'prev-prev-pos': prevprevpos,\n\n            'prev-iob': previob,\n\n            'contains-dash': contains_dash,\n            'contains-dot': contains_dot,\n\n            'all-caps': allcaps,\n            'capitalized': capitalized,\n\n            'prev-all-caps': prevallcaps,\n            'prev-capitalized': prevcapitalized,\n\n            'next-all-caps': nextallcaps,\n            'next-capitalized': nextcapitalized,\n        }\n\n        return f\n\ndef get_dataset_chunker(dataset_file_name):\n    \"\"\"\n    returns DatasetChunker instance with dataset_file_name as training samples\n    `dataset_file_name` = file name of pickled list of CoNLL IOB format sentences\n    \"\"\"\n\n    chunker = DatasetChunker(dataset_file_name)\n\n    return chunker","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:50.855462Z","iopub.execute_input":"2021-06-20T18:54:50.855714Z","iopub.status.idle":"2021-06-20T18:54:50.871707Z","shell.execute_reply.started":"2021-06-20T18:54:50.85569Z","shell.execute_reply":"2021-06-20T18:54:50.870618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_chuncker_accuracy(chunker, test_samples):\n    \"\"\"\n    returns score of the chunker against the gold standard\n    \"\"\"\n    score = chunker.evaluate([\n        conlltags2tree([(w, t, iob) for (w, t), iob in iobs])\n        for iobs in test_samples\n        ])\n    return score.accuracy()\n\ndef get_tagged_sentence(chunker, sentence):\n    \"\"\"\n    returns IOB tagged tree of sentence\n    \"\"\"\n    return chunker.parse(pos_tag(word_tokenize(sentence)))\n\ndef extract_dataset(chunker, sentence):\n    \"\"\"\n    returns all datasets in sentence\n    \"\"\"\n    def tree_filter(tree):\n        return GPE_TAG == tree.label()\n\n    tagged_tree = get_tagged_sentence(chunker, sentence)\n    datasets = list()\n    for subtree in tagged_tree.subtrees(filter=tree_filter):\n        datasets.append(untag(subtree.leaves()))\n    return datasets","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:50.87305Z","iopub.execute_input":"2021-06-20T18:54:50.873324Z","iopub.status.idle":"2021-06-20T18:54:50.886375Z","shell.execute_reply.started":"2021-06-20T18:54:50.8733Z","shell.execute_reply":"2021-06-20T18:54:50.885722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Loading dataset...\")\nchunker = get_dataset_chunker(DATA[0:25])\nprint(\"Done.\")","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:50.887481Z","iopub.execute_input":"2021-06-20T18:54:50.887722Z","iopub.status.idle":"2021-06-20T18:54:51.138407Z","shell.execute_reply.started":"2021-06-20T18:54:50.887698Z","shell.execute_reply":"2021-06-20T18:54:51.13762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting list of publication ids in the test set\ntest_pubs = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\").Id\ntest_pubs\n\n# load submission.csv\nsub = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-20T18:54:51.139476Z","iopub.execute_input":"2021-06-20T18:54:51.139882Z","iopub.status.idle":"2021-06-20T18:54:51.151889Z","shell.execute_reply.started":"2021-06-20T18:54:51.139808Z","shell.execute_reply":"2021-06-20T18:54:51.150933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\n\nfor pub in test_pubs:    \n    print(\"pub:\",pub)\n    \n    f = open(\"../input/coleridgeinitiative-show-us-the-data/test/\" + pub + \".json\")  \n    \n    data = json.load(f)      \n\n    sentences = nltk.tokenize.sent_tokenize(str(data))        \n    \n    predicted_dataset=\"\"\n    dataset=[]\n    final_prediction = \"\"\n    \n    for sentence in sentences: \n        text = clean_sentence(sentence).strip()       \n        dataset = extract_dataset(chunker, text)       \n        \n        if len(dataset)>0:\n            for j in range(len(dataset)):\n                for ds in dataset[j]:\n                    predicted_dataset = predicted_dataset + \" \" + ds\n                    #print(predicted_dataset)\n\n        \n              \n                sub.PredictionString.loc[i] = predicted_dataset.lower().strip()\n    \n    i = i + 1","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:10:02.957355Z","iopub.execute_input":"2021-06-20T19:10:02.957699Z","iopub.status.idle":"2021-06-20T19:10:19.681071Z","shell.execute_reply.started":"2021-06-20T19:10:02.95767Z","shell.execute_reply":"2021-06-20T19:10:19.680057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv',index=False)\nsub","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:10:20.406264Z","iopub.execute_input":"2021-06-20T19:10:20.406573Z","iopub.status.idle":"2021-06-20T19:10:20.415762Z","shell.execute_reply.started":"2021-06-20T19:10:20.406545Z","shell.execute_reply":"2021-06-20T19:10:20.414898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}