{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What is this ?","metadata":{}},{"cell_type":"markdown","source":"In this baseline, we match exactly. ","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport json","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# read data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_of_data = set()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for data_name in df_train[\"cleaned_label\"]:\n    set_of_data.add(data_name.strip())\n    set_of_data.add(data_name.strip().lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set_of_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set_of_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read test data","metadata":{}},{"cell_type":"code","source":"prefix_test = \"/kaggle/input/coleridgeinitiative-show-us-the-data/test/\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = []\nfor dirname, _, filenames in os.walk(prefix_test):\n    for filename in filenames:\n        # print(os.path.join(dirname, filename))\n        test_path.append(os.path.join(dirname, filename))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load Test data","metadata":{}},{"cell_type":"code","source":"pub_list = []\npub_id = []\nfor p in test_path:\n    with open(p, \"r\") as f:\n        pub = json.loads(f.read())\n        pub_list.append(pub)\n        pub_id.append(p.split(\"/\")[-1].split(\".\")[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pub_list[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"format into section","metadata":{}},{"cell_type":"code","source":"section_list = []\nfor pid, pub in zip(pub_id, pub_list):\n    for sec in pub:\n        section_list.append(sec)\n        sec[\"id\"] = pid","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\ntxt = \"The rain in Spain\"\nx = re.search(\"x\", txt)\ny = re.search(\"Spain\", txt)\nprint(x) #this will print an object\nprint(y)\nprint(y.span())\nprint(txt[y.span()[0]:y.span()[1]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# section_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We create clean text for each section list ","metadata":{}},{"cell_type":"code","source":"for sec in section_list:\n    sec[\"clean_text\"] = clean_text(sec[\"text\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We create a dict of result. \n\nAll id must be include in submission.csv, including empty line with only id.\n\n","metadata":{}},{"cell_type":"code","source":"result = {}\nfor sec in section_list:\n    if not result.get(sec[\"id\"], None):\n        result[sec[\"id\"]] = \"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor sec in section_list:\n    for data_name in set_of_data:\n        clean_text = sec[\"clean_text\"]\n        x = re.search(data_name, clean_text)\n        if x: \n            # print(sec[\"id\"], data_name)\n            if not result.get(sec[\"id\"], None) or result.get(sec[\"id\"], None) == \"\":\n                result[sec[\"id\"]] = data_name\n            else:\n                result[sec[\"id\"]] = result[sec[\"id\"]] + \"|\" + data_name\n\n\n                ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pidlist = []\nexcerptlist = []\nfor pid in result.keys():\n    pidlist.append(pid)\n    excerptlist.append(result[pid])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# excerptlist","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We sort excerpt by alphabet value","metadata":{}},{"cell_type":"code","source":"sort_excerptlist = []\nfor ex in excerptlist:\n    ex_str_list = ex.split(\"|\")\n    ex_str_list.sort()\n    n_ex = \"|\".join(ex_str_list)\n    sort_excerptlist.append(n_ex)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sort_excerptlist","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_pd = pd.DataFrame(data={\"Id\":pidlist, \"PredictionString\": sort_excerptlist})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_pd.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_pd.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}