{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport json\nimport glob\nimport re\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATH:\n    base=\"/kaggle/input/coleridgeinitiative-show-us-the-data/\"\n    train=\"/kaggle/input/coleridgeinitiative-show-us-the-data/train/\"\n    test=\"/kaggle/input/coleridgeinitiative-show-us-the-data/test/\"\n    traincsv = base+\"train.csv\"\n    \n    sample_submission=base+\"sample_submission.csv\"\n    submission=\"submission.csv\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from queue import Queue\n\nclass Node(dict):\n        def __init__(self):\n            super().__init__()\n            self.final = False;\n            \n            self.out = set();\n            self.fail = None;\n            \n        def addout(self,out):\n            if type(out) is set:\n                self.out = self.out.union(out)\n            else :\n                self.out.add(out)\n        \n        def addchild(self,alphabet,node = None):\n            self[alphabet] = Node() if node is None else node\n\nclass AC():\n       \n    def __init__(self,patterns):\n        self.patterns = patterns\n        self.head = Node()\n        \n        self.maketrie()\n        self.constructfail()\n        \n    def search(self,sentence):\n        crr = self.head\n        ret = []\n        for c in sentence :\n            while crr is not self.head and c not in crr:\n                crr = crr.fail\n            if c in crr:\n                crr = crr[c]\n            \n            if crr.final:\n                ret.extend(list(crr.out))\n        return ret\n    \n    def maketrie(self):\n        for pattern in self.patterns:\n            crr = self.head\n            for c in pattern :\n                if c not in crr:\n                    crr.addchild(c)\n                crr = crr[c]\n            crr.final = True\n            crr.addout(pattern)\n            \n    def constructfail(self):\n        queue = Queue()\n        self.head.fail = self.head\n        queue.put(self.head)\n        while not queue.empty():\n            crr = queue.get()\n            for nextc in crr:\n                child = crr[nextc]\n                \n                if crr is self.head:\n                    child.fail = self.head\n                else :\n                    f = crr.fail\n                    while f is not self.head and nextc not in f:\n                        f = f.fail\n                    if nextc in f:\n                        f = f[nextc]\n                    child.fail = f\n                \n                child.addout(child.fail.out)\n                child.final |= child.fail.final\n                \n                queue.put(child)\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(PATH.traincsv)\nunq_labels = df_train['cleaned_label'].unique()\nunq_labels = map(lambda x : x.strip(),unq_labels)\n\nac = AC(unq_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ac.search(' oh alzheimer s disease neuroimaging initiative adniiii')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids, predictions =[],[]\n\nfor path in glob.iglob(PATH.test+\"*\"):\n    idx = os.path.basename(path)[:-5]\n    \n    json_file = None\n    with open(path,'r') as file:\n        json_file = json.load(file)\n    \n    pred = []\n    for content in json_file :\n        txt = clean_text(content['text'])\n        pred += ac.search(txt)\n        \n    pred  = list(set(pred))\n    predictions.append(\"|\".join(pred))\n    ids.append(idx)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame.from_dict({'Id':ids,'PredictionString':predictions})\nsubmission.to_csv(PATH.submission, index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}