{"cells":[{"metadata":{},"cell_type":"markdown","source":"A simple baseline: look if any of the training label text appears in the test data text (string matching training labels in test document text) and if they do, assign them to the test documents."},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n\nimport json\nimport re\nimport os\n\nINPUT_PATH = '/kaggle/input/coleridgeinitiative-show-us-the-data'\nWORKING_PATH = '/kaggle/working'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef read_doc(doc_id, split:str = 'train'):\n    path = os.path.join(INPUT_PATH, split, f'{doc_id}.json')\n    return json.loads(open(path).read())\n\n    \ndef row_to_doc_text(row: pd.Series, split:str = 'train') -> str:\n    \n    doc_id = row['Id']\n    doc_title = row['pub_title'] if 'pub_title' in row else ''\n    \n    doc_content = read_doc(doc_id, split)\n    doc_text = ' '.join([section['section_title'] + ' ' + \n                         section['text'] for section in doc_content])\n    \n    return clean_text(doc_title + ' ' + doc_text).strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(INPUT_PATH, 'train.csv'))\n\ntest_df = pd.read_csv(os.path.join(INPUT_PATH, 'sample_submission.csv'))\ntest_df['pub_text'] = test_df.apply(row_to_doc_text, split='test', axis='columns')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_labels = train_df['cleaned_label'].unique()\n\ndef text_to_labels(text, labels):\n    return '|'.join([label for label in labels if label in text]).strip()\n\ntest_df['PredictionString'] = test_df['pub_text'].apply(text_to_labels, labels=unique_labels)\ntest_df.drop('pub_text', inplace=True, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}