{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport json\nimport numpy as np\nimport pandas as pd\nimport spacy","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-16T13:00:09.83851Z","iopub.execute_input":"2021-06-16T13:00:09.838891Z","iopub.status.idle":"2021-06-16T13:00:10.674289Z","shell.execute_reply.started":"2021-06-16T13:00:09.838821Z","shell.execute_reply":"2021-06-16T13:00:10.673249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))\n\nnlp = spacy.load(\"../input/coleridge-ner-chain-v02-c04/coleridge_ner\", disable=[\"tagger\", \"parser\"])\nnlp.max_length = 2e6\nkeep_ents = [\"EVENT\", \"FAC\", \"GPE\", \"LAW\", \"LOC\", \"NORP\", \"ORG\", \"PRODUCT\", \"WORK_OF_ART\"]\n\ndf = pd.read_csv(\"../input/coleridgeextradsets/data_set_800.csv\")\nvocab = df['title'].unique().tolist()","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:00:10.675607Z","iopub.execute_input":"2021-06-16T13:00:10.67588Z","iopub.status.idle":"2021-06-16T13:00:24.61283Z","shell.execute_reply.started":"2021-06-16T13:00:10.675854Z","shell.execute_reply":"2021-06-16T13:00:24.611901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aux_df = []\nfor f in os.listdir(\"../input/coleridgeextradsets\"):\n    if f.startswith(\"SciData\"):\n        temp = pd.read_csv(os.path.join(\"../input/coleridgeextradsets\", f))[[\"Repository Name\", \"Abbreviation\"]]\n        aux_df.append(temp)\naux_df = pd.concat(aux_df, ignore_index=True)\naux_df.loc[:, \"Repository Name\"] = aux_df[\"Repository Name\"].apply(clean_text).str.strip()\naux_vocab = aux_df[\"Repository Name\"].dropna().unique().tolist()\naux_vocab = [v for v in aux_vocab if v not in [\"figshare\", \"intact\", \"massive\", \"pride\", \"nan\"]]\nvocab += aux_vocab\nvocab = list(np.unique(vocab))\nent_vocab = [v for v in vocab if len(v) > 10]","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:00:24.617738Z","iopub.execute_input":"2021-06-16T13:00:24.618095Z","iopub.status.idle":"2021-06-16T13:00:24.674064Z","shell.execute_reply.started":"2021-06-16T13:00:24.61807Z","shell.execute_reply":"2021-06-16T13:00:24.673037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dir = \"../input/coleridgeinitiative-show-us-the-data/test\"\nsub_tups = []\nid_list = []\n\nfor f in os.listdir(test_dir):\n    \n    Id = f[:-5]\n    id_list.append(Id)\n    with open(os.path.join(test_dir, f)) as rf:\n        txt = json.load(rf)\n    \n    for sec in txt:\n        \n        if len(sec[\"text\"]) < 2e6:\n            doc = nlp(sec[\"text\"])\n            for e in doc.ents:\n                if e.label_ == \"DATASET\":\n                    sub_tups.append((Id, clean_text(e.text), \"NLP\"))\n                if e.label_ in keep_ents:\n                    e_txt = clean_text(e.text)\n                    if any([jaccard(e_txt, v) > 0.95 for v in ent_vocab]):\n                        sub_tups.append((Id, v, \"match\")) \n                    \n        txt_cln = clean_text(sec[\"text\"])\n        for v in vocab:\n            if v in txt_cln:\n                sub_tups.append((Id, v, \"naive\"))                   ","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:07:24.796364Z","iopub.execute_input":"2021-06-09T13:07:24.796685Z","iopub.status.idle":"2021-06-09T13:07:39.978493Z","shell.execute_reply.started":"2021-06-09T13:07:24.796658Z","shell.execute_reply":"2021-06-09T13:07:39.977623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_0 = pd.DataFrame(sub_tups, columns=[\"Id\", \"PredictionString\", \"Type\"])\nsub_nlp = sub_0[sub_0[\"Type\"] == \"NLP\"]\nsub_naive = sub_0[sub_0[\"Type\"] == \"naive\"]\nsub_disj = sub_nlp.merge(sub_naive, on=[\"Id\", \"PredictionString\"], how=\"left\")\nsub_disj = sub_disj[sub_disj[\"Type_y\"].isna()][[\"Id\", \"PredictionString\"]]\nsub_1 = pd.concat([sub_naive.drop(\"Type\", 1), sub_disj], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:10:52.366091Z","iopub.execute_input":"2021-06-09T13:10:52.366667Z","iopub.status.idle":"2021-06-09T13:10:52.379433Z","shell.execute_reply.started":"2021-06-09T13:10:52.366619Z","shell.execute_reply":"2021-06-09T13:10:52.378693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_match = sub_0[sub_0[\"Type\"] == \"match\"]\nsub_1.loc[:, \"Type\"] = \"stage_1\"\nsub_disj = sub_match.merge(sub_1, on=[\"Id\", \"PredictionString\"], how=\"left\")\nsub_disj = sub_disj[sub_disj[\"Type_y\"].isna()][[\"Id\", \"PredictionString\"]]\nsub_df = pd.concat([sub_1.drop(\"Type\", 1), sub_disj], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:11:15.121032Z","iopub.execute_input":"2021-06-09T13:11:15.121352Z","iopub.status.idle":"2021-06-09T13:11:15.132856Z","shell.execute_reply.started":"2021-06-09T13:11:15.121322Z","shell.execute_reply":"2021-06-09T13:11:15.131919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_mentions = 1\nsub_df = sub_df.groupby([\"Id\", \"PredictionString\"])[\"Id\"].count().rename(\"N\").reset_index()\nsub_df.loc[:, \"N\"] = np.minimum(min_mentions, sub_df[\"N\"].values)\nsub_df.loc[:, \"PredictionString\"] = sub_df.apply(lambda x: \"|\".join(np.tile(x[\"PredictionString\"], x[\"N\"])), axis=1)\nsub_df = sub_df.groupby(\"Id\")[\"PredictionString\"].apply(lambda x: \"|\".join(x)).reset_index()\nid_df = pd.DataFrame(id_list, columns=[\"Id\"])\nsub_df = id_df.merge(sub_df, on=\"Id\", how=\"left\").fillna(\"\")\nsub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T11:22:17.487058Z","iopub.execute_input":"2021-06-08T11:22:17.487617Z","iopub.status.idle":"2021-06-08T11:22:17.514243Z","shell.execute_reply.started":"2021-06-08T11:22:17.487584Z","shell.execute_reply":"2021-06-08T11:22:17.513236Z"},"trusted":true},"execution_count":null,"outputs":[]}]}