{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Test data are NOT fully labeled (!!)\n\nThis notebook is the replication code or the discussion at:\nhttps://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/233170\n\nImport some modules and define some utility functions.","metadata":{"_uuid":"16191143-1c52-4e6a-9b97-632c2ba838cb","_cell_guid":"5d25b3d6-f85c-4f6f-816d-060775ce35d7","trusted":true}},{"cell_type":"code","source":"import os\nimport glob\nimport regex\nimport pandas as pd\nfrom tqdm import tqdm","metadata":{"_uuid":"0f52a805-25bb-4080-b07c-809e407a51ae","_cell_guid":"c62a50e7-21dd-446a-9f9c-3bc8f5c1a312","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"_uuid":"fdaa640a-17e1-4b71-a036-9ec5ccd97807","_cell_guid":"677ad116-fbd6-409f-81e8-099f4fdff98f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def import_json(dataset):\n    files = glob.glob(f\"../input/coleridgeinitiative-show-us-the-data/{dataset}/*.json\")\n    data = {}\n    for file in tqdm(files):\n        df = pd.read_json(file)\n        pub_id = os.path.basename(file)[:-5]\n        data[pub_id] = ' '.join(df.text)\n    return data","metadata":{"_uuid":"ae212b66-5e2b-41af-a0de-3b8c33aba8f4","_cell_guid":"b10f074f-b46a-4bf9-8b97-c0a6e4dde5ab","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = import_json('test')","metadata":{"_uuid":"a437bdd4-a233-40dc-b2b6-96728f73a63d","_cell_guid":"cc930de8-a0b5-4831-ba3a-fc9b407abaad","jupyter":{"outputs_hidden":false},"collapsed":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add exact matching","metadata":{"_uuid":"c0c14fce-c564-4cd8-bb39-46e4639e98bb","_cell_guid":"dc02efbf-2a60-44ba-86ec-c9954106bf79","trusted":true}},{"cell_type":"markdown","source":"Existing datasets.","metadata":{"_uuid":"802c0acf-b26b-456d-b8a7-fc2c11ee5815","_cell_guid":"df46633e-252a-4d20-89fc-f1159e4b628a","trusted":true}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ndatasets = list(df['dataset_label']) + list(df['dataset_title']) + list(df['cleaned_label'])\ndatasets = {clean_text(d) for d in datasets}","metadata":{"_uuid":"b468eebb-2d77-4d76-a258-1f893b5bf716","_cell_guid":"28537485-2c7d-4315-bbae-970e6f4f37c4","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For each publication, add to the predictions the datasets that appear in the text via exact matching.","metadata":{"_uuid":"8d82e7fb-b016-4476-9599-898266ac47a3","_cell_guid":"2598f226-d170-47da-a270-4d1572f4723c","trusted":true}},{"cell_type":"code","source":"pred = []\nfor pub_id in tqdm(test.keys()):\n    # clean text\n    text = clean_text(test[pub_id])\n    # list of matching strings\n    matches = []    \n    for dataset in datasets:\n        if dataset in text:\n        #if regex.search(f\"\\\\b{dataset}\\\\b\", text):\n            matches += [dataset]\n    if 'assessment of educational progress' in text:\n        matches += ['ThisStringWillNeverMatch']\n    # append prediction\n    pred.append({'Id': pub_id, 'PredictionString': \"|\".join(matches)})","metadata":{"_uuid":"1d02a9c8-5d71-4f27-83a3-bfee34843487","_cell_guid":"8375bfbe-4356-4a14-942f-fd7fe7c43d27","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reshape as requested by the submission. ","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame(pred)\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Save to file and submit.","metadata":{"_uuid":"ce0204bb-dccf-437f-85c4-002cb5208175","_cell_guid":"f93de2c8-1326-4f7f-b28d-9bafe1cd3a8c","trusted":true}},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"_uuid":"66a9c27f-4471-4b2f-aa1f-7971450ddede","_cell_guid":"74420d04-b518-40b1-aabb-3deea0d4fd28","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}