{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport json\nimport numpy as np\nimport pandas as pd\nimport spacy","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-16T12:48:16.027444Z","iopub.execute_input":"2021-06-16T12:48:16.028058Z","iopub.status.idle":"2021-06-16T12:48:16.857169Z","shell.execute_reply.started":"2021-06-16T12:48:16.027932Z","shell.execute_reply":"2021-06-16T12:48:16.855978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\nnlp = spacy.load(\"../input/coleridge-ner-chain-v02-c04/coleridge_ner\", disable=[\"tagger\", \"parser\"])\nnlp.max_length = 2e6\n\ndf = pd.read_csv(\"../input/coleridgeextradsets/data_set_800.csv\")\nvocab = df['title'].unique().tolist()","metadata":{"execution":{"iopub.status.busy":"2021-06-16T12:57:13.277295Z","iopub.execute_input":"2021-06-16T12:57:13.27764Z","iopub.status.idle":"2021-06-16T12:57:20.287261Z","shell.execute_reply.started":"2021-06-16T12:57:13.277611Z","shell.execute_reply":"2021-06-16T12:57:20.285981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aux_df = []\nfor f in os.listdir(\"../input/coleridgeextradsets\"):\n    if f.startswith(\"SciData\"):\n        temp = pd.read_csv(os.path.join(\"../input/coleridgeextradsets\", f))[[\"Repository Name\", \"Abbreviation\"]]\n        aux_df.append(temp)\naux_df = pd.concat(aux_df, ignore_index=True)\naux_df.loc[:, \"Repository Name\"] = aux_df[\"Repository Name\"].apply(clean_text).str.strip()\naux_vocab = aux_df[\"Repository Name\"].dropna().unique().tolist()\naux_vocab = [v for v in aux_vocab if v not in [\"figshare\", \"intact\", \"massive\", \"pride\", \"nan\"]]\nvocab += aux_vocab\nvocab = list(np.unique(vocab))","metadata":{"execution":{"iopub.status.busy":"2021-06-16T12:58:32.512378Z","iopub.execute_input":"2021-06-16T12:58:32.512884Z","iopub.status.idle":"2021-06-16T12:58:32.563515Z","shell.execute_reply.started":"2021-06-16T12:58:32.512851Z","shell.execute_reply":"2021-06-16T12:58:32.562466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dir = \"../input/coleridgeinitiative-show-us-the-data/test\"\nsub_tups = []\nid_list = []\n\nfor f in os.listdir(test_dir):\n    \n    Id = f[:-5]\n    id_list.append(Id)\n    with open(os.path.join(test_dir, f)) as rf:\n        txt = json.load(rf)\n    \n    for sec in txt:\n        \n        if len(sec[\"text\"]) < 2e6:\n            doc = nlp(sec[\"text\"])\n            for e in doc.ents:\n                if e.label_ == \"DATASET\":\n                    sub_tups.append((Id, clean_text(e.text), \"NLP\"))\n                    \n        txt_cln = clean_text(sec[\"text\"])\n        for v in vocab:\n            if v in txt_cln:\n                sub_tups.append((Id, v, \"naive\"))                   ","metadata":{"execution":{"iopub.status.busy":"2021-06-08T11:12:55.19194Z","iopub.execute_input":"2021-06-08T11:12:55.192276Z","iopub.status.idle":"2021-06-08T11:12:58.340542Z","shell.execute_reply.started":"2021-06-08T11:12:55.192247Z","shell.execute_reply":"2021-06-08T11:12:58.339562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_0 = pd.DataFrame(sub_tups, columns=[\"Id\", \"PredictionString\", \"Type\"])\nsub_nlp = sub_0[sub_0[\"Type\"] == \"NLP\"]\nsub_naive = sub_0[sub_0[\"Type\"] == \"naive\"]\nsub_disj = sub_nlp.merge(sub_naive, on=[\"Id\", \"PredictionString\"], how=\"left\")\nsub_disj = sub_disj[sub_disj[\"Type_y\"].isna()][[\"Id\", \"PredictionString\"]]\nsub_df = pd.concat([sub_naive.drop(\"Type\", 1), sub_disj], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T11:22:45.811856Z","iopub.execute_input":"2021-06-08T11:22:45.812417Z","iopub.status.idle":"2021-06-08T11:22:45.828557Z","shell.execute_reply.started":"2021-06-08T11:22:45.812385Z","shell.execute_reply":"2021-06-08T11:22:45.827608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_mentions = 2\nsub_df = sub_df.groupby([\"Id\", \"PredictionString\"])[\"Id\"].count().rename(\"N\").reset_index()\nsub_df.loc[:, \"N\"] = np.minimum(min_mentions, sub_df[\"N\"].values)\nsub_df.loc[:, \"PredictionString\"] = sub_df.apply(lambda x: \"|\".join(np.tile(x[\"PredictionString\"], x[\"N\"])), axis=1)\nsub_df = sub_df.groupby(\"Id\")[\"PredictionString\"].apply(lambda x: \"|\".join(x)).reset_index()\nid_df = pd.DataFrame(id_list, columns=[\"Id\"])\nsub_df = id_df.merge(sub_df, on=\"Id\", how=\"left\").fillna(\"\")\nsub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T11:22:17.487058Z","iopub.execute_input":"2021-06-08T11:22:17.487617Z","iopub.status.idle":"2021-06-08T11:22:17.514243Z","shell.execute_reply.started":"2021-06-08T11:22:17.487584Z","shell.execute_reply":"2021-06-08T11:22:17.513236Z"},"trusted":true},"execution_count":null,"outputs":[]}]}