{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport nltk\nimport textblob\nimport collections\nimport json\nimport tqdm\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntraining_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data['dataset_label'].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I am goint to use *Pointwise Mutual Information* to try to extract key phrases from the documents, and then match the key phrases to the dataset names.\nPointwise Mutual Information will be calculated on pairs of successive words from within a sentence, after removal of stopwords.","metadata":{}},{"cell_type":"code","source":"individual_counts = collections.defaultdict(float)\npair_counts = collections.defaultdict(float)\nn_words = 0.0\nn_sentences = 0.0\nstopwords = nltk.corpus.stopwords.words('english')\nfor filename in tqdm.tqdm(os.listdir('../input/coleridgeinitiative-show-us-the-data/train')):\n    with open('/'.join(('../input/coleridgeinitiative-show-us-the-data/train',filename))) as data:\n        article = json.load(data)\n        text = '\\n'.join(['\\n'.join((section['section_title'],section['text']))\n                          for section in article])\n        sentences = textblob.TextBlob(text).sentences\n        n_sentences += len(sentences)\n        for sentence in sentences:\n            words = [str(word) for word in sentence.lower().words\n                            if str(word) not in stopwords]\n            if len(words)>0:\n                n_words += len(words)\n                for (i,word) in enumerate(words[:-1]):\n                    individual_counts[word] +=1.0\n                    pair_counts[(word,words[i+1])] += 1.0\n                individual_counts[words[-1]]+=1.0\n            else:\n                n_sentences -= 1\n\npmi = pd.Series({(word0,word1):(n * ((n_words - n_sentences)**2.0))/(n_words * individual_counts[word0] * individual_counts[word1])\n                 for ((word0,word1),n) in tqdm.tqdm(pair_counts.items())\n                if n > 1}).apply(np.log2)\n\npmi.quantile(np.linspace(0.0,1.0,101)).plot.line()\n            \n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pmi[pmi>=3]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let us find some candidate sequences from each document. These are sequences of words from within a sentence where (not counting stopwords) each pair of successive words has a Pointwise Mutual Information > 3 bits.","metadata":{}},{"cell_type":"code","source":"def candidates(document):\n    text = '\\n'.join(['\\n'.join((section['section_title'],section['text']))\n                     for section in article])\n    for sentence in textblob.TextBlob(text).sentences:\n        prev_word = None\n        prev_index = None\n        span = []\n        words = [str(word) for word in sentence.lower().words]\n        for (i,word) in enumerate(words):\n            if word not in stopwords:\n                score = pmi.get((prev_word,word),0)\n                if score >3:\n                    span.append(prev_word)\n                elif len(span) > 0:\n                    span.append(prev_word)\n                    yield (span,' '.join(words[prev_index:i]))\n                    span = []\n                    prev_index = i\n                else:\n                    prev_index = i\n                prev_word = word\n        if len(span) > 0:\n            span.append(prev_word)\n            yield  (span,' '.join(words[prev_index:]))\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/coleridgeinitiative-show-us-the-data/train/0007f880-0a9b-492d-9a58-76eb0b0e0bd7.json') as doc:\n    for (span,string) in candidates(json.load(doc)):\n        print(span)\n        print(string)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LabelFilter(object):\n    \n    def __init__(self,data):\n        self.labels = {tuple([word for word in label.split()\n                        if word not in stopwords])\n                       for label in data['cleaned_label'].unique()}\n        \n    def match(self,candidate,label):\n        word = label[0]\n        result = word in candidate\n        if result:\n            n = candidate.index(word)\n            result = tuple(candidate[n:n+len(label)]) == label\n        return result\n    \n    def __call__(self,candidate):\n        return any((self.match(candidate,label)\n                   for label in self.labels))     \n    \nlabel_filter = LabelFilter(training_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/coleridgeinitiative-show-us-the-data/train/0007f880-0a9b-492d-9a58-76eb0b0e0bd7.json') as doc:\n    print([string for (span,string) in candidates(doc)\n          if label_filter(string)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}