{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport os\nimport json\nimport re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-18T16:37:43.030136Z","iopub.execute_input":"2021-06-18T16:37:43.030787Z","iopub.status.idle":"2021-06-18T16:37:43.041921Z","shell.execute_reply.started":"2021-06-18T16:37:43.030624Z","shell.execute_reply":"2021-06-18T16:37:43.040535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read in the csvs\ndf = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\nhandlabeled = pd.read_csv('../input/coleridgehandlabeled/labels.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:37:56.197088Z","iopub.execute_input":"2021-06-18T16:37:56.197711Z","iopub.status.idle":"2021-06-18T16:37:56.290491Z","shell.execute_reply.started":"2021-06-18T16:37:56.197667Z","shell.execute_reply":"2021-06-18T16:37:56.289657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def chunk_text(full_text, length=250, overlap=25):\n    full_text = full_text.split()\n    text_len = len(full_text)\n    results = []\n    i = 0\n    while i < text_len:\n        results.append(' '.join(full_text[i:i+length]))\n        i = i + length - overlap\n    return results","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:37:57.628911Z","iopub.execute_input":"2021-06-18T16:37:57.629693Z","iopub.status.idle":"2021-06-18T16:37:57.634640Z","shell.execute_reply.started":"2021-06-18T16:37:57.629648Z","shell.execute_reply":"2021-06-18T16:37:57.633915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_label_text(txt):\n    txt = re.sub('[^A-Za-z0-9()]+', ' ', str(txt)).strip()\n    toks = txt.split()\n    if '(' in toks[-1]:\n        txt = ' '.join(toks[:-1])\n    return re.sub('\\s+', ' ', str(txt)).strip()\n\ndef clean_text(txt):\n    txt = re.sub('[^A-Za-z0-9()]+', ' ', str(txt)).strip()\n    return re.sub('\\s+', ' ', str(txt)).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:37:58.024233Z","iopub.execute_input":"2021-06-18T16:37:58.024903Z","iopub.status.idle":"2021-06-18T16:37:58.033051Z","shell.execute_reply.started":"2021-06-18T16:37:58.024841Z","shell.execute_reply":"2021-06-18T16:37:58.032149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = handlabeled.labels.unique().tolist()\ncleaned_labels = set([])\nfor label in labels:\n    if label not in cleaned_labels:\n        cleaned = clean_label_text(label)\n        cleaned_labels.add(cleaned)\ncleaned_labels = sorted(cleaned_labels, key=len, reverse=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T16:43:49.630850Z","iopub.execute_input":"2021-06-18T16:43:49.631487Z","iopub.status.idle":"2021-06-18T16:43:49.654207Z","shell.execute_reply.started":"2021-06-18T16:43:49.631433Z","shell.execute_reply":"2021-06-18T16:43:49.653017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def is_subset(element, items):\n    for item in items:\n        if element in item:\n            return True\n    return False","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:43:20.229192Z","iopub.execute_input":"2021-06-17T23:43:20.229655Z","iopub.status.idle":"2021-06-17T23:43:20.234712Z","shell.execute_reply.started":"2021-06-17T23:43:20.229627Z","shell.execute_reply":"2021-06-17T23:43:20.233217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_actual_label = set([])\nid_input = df['Id'].unique()\ndf = df.set_index('Id')\ny_true = {}\nfreq = {}\nresults = []\nfor _id in tqdm(id_input):\n    df_labels = df.loc[_id].dataset_label\n    if type(df_labels) == str:\n        y_true[_id] = [df_labels]\n    else:\n        y_true[_id] = df_labels.tolist()\n    json_path = os.path.join(train_files_path, (_id+'.json'))\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        full_text = ''\n        for data in json_decode: # for each section of the document\n            full_text += ' ' + data['section_title'] + ' ' + data['text']\n        full_text = clean_text(full_text)\n        for label in cleaned_labels:\n            if label in full_text and label not in y_true[_id] and label:\n                if not is_subset(label, y_true[_id]):\n                    y_true[_id].append(label.strip())\n        chunks = chunk_text(full_text)\n        for chunk in chunks:\n            chunk_labels = []\n            for cleaned in sorted(y_true[_id], key=len, reverse=True):\n                if cleaned in chunk and not is_subset(cleaned, chunk_labels):\n                    chunk_labels.append(cleaned.strip())\n                    if cleaned in freq:\n                        freq[cleaned.strip()] += 1\n                    else:\n                        freq[cleaned.strip()] = 1\n            results.append((_id, '|'.join(chunk_labels), chunk))\n            if chunk_labels:\n                is_actual_label.update(chunk_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T23:43:22.023944Z","iopub.execute_input":"2021-06-17T23:43:22.024443Z","iopub.status.idle":"2021-06-17T23:51:04.68813Z","shell.execute_reply.started":"2021-06-17T23:43:22.024405Z","shell.execute_reply":"2021-06-17T23:51:04.686951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df = pd.DataFrame(results, columns=['ids', 'labels', 'chunks'])\nresult_df.to_csv('data.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T00:03:28.552445Z","iopub.execute_input":"2021-06-17T00:03:28.553009Z","iopub.status.idle":"2021-06-17T00:03:29.194915Z","shell.execute_reply.started":"2021-06-17T00:03:28.552971Z","shell.execute_reply":"2021-06-17T00:03:29.1935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('frequencies.json', 'w') as f:\n    json.dump(freq, f)\n\nwith open('y_true.json', 'w') as f:\n    json.dump(y_true, f)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T00:03:29.196876Z","iopub.execute_input":"2021-06-17T00:03:29.197267Z","iopub.status.idle":"2021-06-17T00:03:29.206501Z","shell.execute_reply.started":"2021-06-17T00:03:29.197222Z","shell.execute_reply":"2021-06-17T00:03:29.205535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"labels.csv\", 'w') as file:\n    file.write('labels\\n')\n    for row in list(is_actual_label):\n        file.write(row+'\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-17T00:03:29.522956Z","iopub.execute_input":"2021-06-17T00:03:29.523514Z","iopub.status.idle":"2021-06-17T00:03:29.528174Z","shell.execute_reply.started":"2021-06-17T00:03:29.523463Z","shell.execute_reply":"2021-06-17T00:03:29.527426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}