{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport glob\nimport json\nimport re\nimport plotly.express as exp\nimport os\nimport numpy as np\nfrom tqdm.autonotebook import tqdm\n\ndef jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":1.446271,"end_time":"2021-06-02T03:57:52.142133","exception":false,"start_time":"2021-06-02T03:57:50.695862","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:01.289692Z","iopub.execute_input":"2021-06-11T04:07:01.290268Z","iopub.status.idle":"2021-06-11T04:07:01.299847Z","shell.execute_reply.started":"2021-06-11T04:07:01.290226Z","shell.execute_reply":"2021-06-11T04:07:01.298201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import transformers\nimport torch.nn as nn\nimport torch","metadata":{"execution":{"iopub.status.busy":"2021-06-11T04:07:01.529701Z","iopub.execute_input":"2021-06-11T04:07:01.530098Z","iopub.status.idle":"2021-06-11T04:07:01.537232Z","shell.execute_reply.started":"2021-06-11T04:07:01.530062Z","shell.execute_reply":"2021-06-11T04:07:01.536379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\n","metadata":{"papermill":{"duration":0.028942,"end_time":"2021-06-02T03:57:52.193009","exception":false,"start_time":"2021-06-02T03:57:52.164067","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:20.814731Z","iopub.execute_input":"2021-06-11T04:07:20.815078Z","iopub.status.idle":"2021-06-11T04:07:20.820453Z","shell.execute_reply.started":"2021-06-11T04:07:20.815048Z","shell.execute_reply":"2021-06-11T04:07:20.819351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"papermill":{"duration":0.184867,"end_time":"2021-06-02T03:57:52.397374","exception":false,"start_time":"2021-06-02T03:57:52.212507","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:21.819517Z","iopub.execute_input":"2021-06-11T04:07:21.819868Z","iopub.status.idle":"2021-06-11T04:07:21.961181Z","shell.execute_reply.started":"2021-06-11T04:07:21.819838Z","shell.execute_reply":"2021-06-11T04:07:21.960355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = \"../input/coleridgeinitiative-show-us-the-data/train\"\ntrain_files = train.Id.to_numpy()","metadata":{"papermill":{"duration":0.034883,"end_time":"2021-06-02T03:57:52.454021","exception":false,"start_time":"2021-06-02T03:57:52.419138","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:22.594523Z","iopub.execute_input":"2021-06-11T04:07:22.594839Z","iopub.status.idle":"2021-06-11T04:07:22.602293Z","shell.execute_reply.started":"2021-06-11T04:07:22.59481Z","shell.execute_reply":"2021-06-11T04:07:22.601325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset_phrase(entire):\n    words = entire.split()\n    tmp = []\n    prop = []\n    invalid_w = ['the','using']\n    for i in range(len(words)-2):\n        if words[i].lower() in invalid_w:\n            continue\n        elif words[i][0].isupper():\n            tmp.append(words[i]+\" \")\n        elif len(tmp)==0 :\n            continue\n        elif words[i+1][0].isupper():\n            tmp.append(words[i])\n        elif (len(tmp)<=2):\n            tmp = []\n        else:\n            wor = re.split(\"[.,:?]\",\"\".join(tmp)[:-1])\n            for wo in wor:\n                if len(wo.split())<=2:\n                    continue\n                elif wo[0]==\" \":\n                    prop.append(clean_text(wo[1:]))\n                else:\n                    prop.append(clean_text(wo))\n            tmp = []\n    return prop","metadata":{"papermill":{"duration":0.032836,"end_time":"2021-06-02T03:57:52.506433","exception":false,"start_time":"2021-06-02T03:57:52.473597","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:23.534745Z","iopub.execute_input":"2021-06-11T04:07:23.535079Z","iopub.status.idle":"2021-06-11T04:07:23.547702Z","shell.execute_reply.started":"2021-06-11T04:07:23.535038Z","shell.execute_reply":"2021-06-11T04:07:23.546848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datasets = train.cleaned_label.unique()","metadata":{"papermill":{"duration":0.03111,"end_time":"2021-06-02T03:57:52.557381","exception":false,"start_time":"2021-06-02T03:57:52.526271","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:24.274224Z","iopub.execute_input":"2021-06-11T04:07:24.274556Z","iopub.status.idle":"2021-06-11T04:07:24.280574Z","shell.execute_reply.started":"2021-06-11T04:07:24.274525Z","shell.execute_reply":"2021-06-11T04:07:24.27945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_dsets = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')","metadata":{"papermill":{"duration":0.158195,"end_time":"2021-06-02T03:57:52.734673","exception":false,"start_time":"2021-06-02T03:57:52.576478","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:25.154187Z","iopub.execute_input":"2021-06-11T04:07:25.15455Z","iopub.status.idle":"2021-06-11T04:07:25.174139Z","shell.execute_reply.started":"2021-06-11T04:07:25.154519Z","shell.execute_reply":"2021-06-11T04:07:25.173266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered = []\nfor x in extra_dsets.title:\n    if len(x.split())>1:\n        filtered.append(clean_text(x))","metadata":{"papermill":{"duration":0.07768,"end_time":"2021-06-02T03:57:52.833106","exception":false,"start_time":"2021-06-02T03:57:52.755426","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:26.004486Z","iopub.execute_input":"2021-06-11T04:07:26.004812Z","iopub.status.idle":"2021-06-11T04:07:26.025721Z","shell.execute_reply.started":"2021-06-11T04:07:26.004782Z","shell.execute_reply":"2021-06-11T04:07:26.024851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datasets = np.hstack([train_datasets,np.array(filtered)])","metadata":{"papermill":{"duration":0.11446,"end_time":"2021-06-02T03:57:52.968056","exception":false,"start_time":"2021-06-02T03:57:52.853596","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:26.479208Z","iopub.execute_input":"2021-06-11T04:07:26.479557Z","iopub.status.idle":"2021-06-11T04:07:26.486688Z","shell.execute_reply.started":"2021-06-11T04:07:26.479527Z","shell.execute_reply":"2021-06-11T04:07:26.485488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datasets = np.unique(train_datasets)","metadata":{"papermill":{"duration":0.078539,"end_time":"2021-06-02T03:57:53.066219","exception":false,"start_time":"2021-06-02T03:57:52.98768","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:27.119322Z","iopub.execute_input":"2021-06-11T04:07:27.119693Z","iopub.status.idle":"2021-06-11T04:07:27.126799Z","shell.execute_reply.started":"2021-06-11T04:07:27.119661Z","shell.execute_reply":"2021-06-11T04:07:27.125901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_datasets)","metadata":{"papermill":{"duration":0.030133,"end_time":"2021-06-02T03:57:53.115814","exception":false,"start_time":"2021-06-02T03:57:53.085681","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:27.494363Z","iopub.execute_input":"2021-06-11T04:07:27.494719Z","iopub.status.idle":"2021-06-11T04:07:27.502392Z","shell.execute_reply.started":"2021-06-11T04:07:27.494688Z","shell.execute_reply":"2021-06-11T04:07:27.5013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_dataset(text):\n    f = []\n    for x in train_datasets:\n        count = text.count(x)\n        if count>0:\n            f.append(x)\n    return f   ","metadata":{"papermill":{"duration":0.029712,"end_time":"2021-06-02T03:57:53.164997","exception":false,"start_time":"2021-06-02T03:57:53.135285","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:28.195003Z","iopub.execute_input":"2021-06-11T04:07:28.195384Z","iopub.status.idle":"2021-06-11T04:07:28.200312Z","shell.execute_reply.started":"2021-06-11T04:07:28.195331Z","shell.execute_reply":"2021-06-11T04:07:28.199142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_dataset_rule(entire):\n    phrases = get_dataset_phrase(entire)\n    useful = ['dataset','survey','data','database','study','atlas','collection','sequence','sequences']\n    f= []\n    for p in phrases:\n        if p==\"\":\n            continue\n        fin = p.split()\n        for u in useful:\n            if u in fin:\n                f.append(p)\n                break\n    ans = pd.DataFrame(f).value_counts()\n    f = [z[0] for z in ans.keys()[:2]]\n    return f","metadata":{"papermill":{"duration":0.030233,"end_time":"2021-06-02T03:57:53.215003","exception":false,"start_time":"2021-06-02T03:57:53.18477","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:29.289205Z","iopub.execute_input":"2021-06-11T04:07:29.289569Z","iopub.status.idle":"2021-06-11T04:07:29.297281Z","shell.execute_reply.started":"2021-06-11T04:07:29.289537Z","shell.execute_reply":"2021-06-11T04:07:29.294794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"../input/coleridgeinitiative-show-us-the-data/test\"","metadata":{"papermill":{"duration":0.027165,"end_time":"2021-06-02T03:57:53.261772","exception":false,"start_time":"2021-06-02T03:57:53.234607","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:33.189782Z","iopub.execute_input":"2021-06-11T04:07:33.190105Z","iopub.status.idle":"2021-06-11T04:07:33.193844Z","shell.execute_reply.started":"2021-06-11T04:07:33.190071Z","shell.execute_reply":"2021-06-11T04:07:33.192858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files = os.listdir(test_path)","metadata":{"papermill":{"duration":0.040924,"end_time":"2021-06-02T03:57:53.322148","exception":false,"start_time":"2021-06-02T03:57:53.281224","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:34.370572Z","iopub.execute_input":"2021-06-11T04:07:34.370982Z","iopub.status.idle":"2021-06-11T04:07:34.642328Z","shell.execute_reply.started":"2021-06-11T04:07:34.370941Z","shell.execute_reply":"2021-06-11T04:07:34.641445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = []\nfor file in tqdm(test_files):\n    prediction.append(\"\")","metadata":{"papermill":{"duration":4.136489,"end_time":"2021-06-02T03:57:57.478241","exception":false,"start_time":"2021-06-02T03:57:53.341752","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:34.934401Z","iopub.execute_input":"2021-06-11T04:07:34.934735Z","iopub.status.idle":"2021-06-11T04:07:34.987819Z","shell.execute_reply.started":"2021-06-11T04:07:34.934706Z","shell.execute_reply":"2021-06-11T04:07:34.986892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_files = [f for f in train_files]\n# submission = pd.DataFrame([test_files]).T\n# submission[\"Prediction\"] = \"\"\n# submission.columns  = ['Id', 'PredictionString']","metadata":{"papermill":{"duration":0.02743,"end_time":"2021-06-02T03:57:57.526807","exception":false,"start_time":"2021-06-02T03:57:57.499377","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:40.089853Z","iopub.execute_input":"2021-06-11T04:07:40.090209Z","iopub.status.idle":"2021-06-11T04:07:40.096169Z","shell.execute_reply.started":"2021-06-11T04:07:40.090174Z","shell.execute_reply":"2021-06-11T04:07:40.095165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files = [f[:-5] for f in test_files]\nsubmission = pd.DataFrame([test_files,prediction]).T\nsubmission.columns  = ['Id', 'PredictionString']","metadata":{"papermill":{"duration":0.034928,"end_time":"2021-06-02T03:57:57.582156","exception":false,"start_time":"2021-06-02T03:57:57.547228","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-11T04:07:40.434205Z","iopub.execute_input":"2021-06-11T04:07:40.434583Z","iopub.status.idle":"2021-06-11T04:07:41.535421Z","shell.execute_reply.started":"2021-06-11T04:07:40.434552Z","shell.execute_reply":"2021-06-11T04:07:41.534592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetFinder(nn.Module):\n    \n    def __init__(self,params):\n        super().__init__()\n        self.model = transformers.AutoModel.from_pretrained(\"../input/scibert-huggingface/coleridge-scibert-models/output\")\n        for param in self.model.parameters():\n            param.requires_grad=False\n        self.lstm = torch.nn.LSTM(input_size=768,hidden_size=params['hid_size'],bidirectional=True,batch_first=True)\n        self.fc = torch.nn.Linear(params['hid_size']*2,1)\n        self.e=0\n    def forward(self,inp):\n        inp,_ = self.lstm((self.model(**inp).last_hidden_state))\n        inp = self.fc(inp).squeeze(2)\n        return torch.sigmoid(inp)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T04:07:45.064519Z","iopub.execute_input":"2021-06-11T04:07:45.064867Z","iopub.status.idle":"2021-06-11T04:07:45.071737Z","shell.execute_reply.started":"2021-06-11T04:07:45.064835Z","shell.execute_reply":"2021-06-11T04:07:45.070792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = transformers.AutoTokenizer.from_pretrained('../input/scibert-huggingface/coleridge-scibert-models/output')\nDEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu'\nM_COUNT=5\n\nparams = {\n    'lr':0.001,\n    'loss_func':torch.nn.BCELoss(),\n    'hid_size':512\n}\n\nmodels = [DatasetFinder(params) for i in range(M_COUNT)]\nfor i in range(M_COUNT):\n    models[i].load_state_dict(torch.load(f'../input/processed-train-data-sentence-segmentaion/model_best_{i}.state'))\n    for params in models[i].parameters():\n        params.requires_grad=False\n    models[i].eval()\n    models[i] = models[i].to(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T04:07:46.219642Z","iopub.execute_input":"2021-06-11T04:07:46.220032Z","iopub.status.idle":"2021-06-11T04:08:33.101859Z","shell.execute_reply.started":"2021-06-11T04:07:46.219994Z","shell.execute_reply":"2021-06-11T04:08:33.100941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_dataset_ann(text):\n    t = text.split('.')\n    p = 0\n    inp_id = []\n    attention_mask = []\n    for x in t:\n        com = tokenizer(x,max_length=30,padding=\"max_length\",truncation=True)\n        inp_id.append(torch.tensor(com[\"input_ids\"],dtype=torch.long).view(1,-1))\n        attention_mask.append(torch.tensor(com[\"attention_mask\"],dtype=torch.long).view(1,-1))\n#     ids = np.random.choice(np.arange(len(inp_id)),BATCH_SIZE)\n#     if(len(inp_id)<=BATCH_SIZE):\n#         ids = np.arange(len(inp_id))\n    inp = {\"input_ids\":torch.cat(inp_id,0).to(DEVICE),\n                 \"attention_mask\":torch.cat(attention_mask,0).to(DEVICE)}\n    out = None\n    for model in models:\n        if out is None:\n            out = model(inp).detach().cpu()\n        else:\n            out = out + model(inp).detach().cpu().numpy()\n    out = out/M_COUNT\n    ans = (inp['input_ids'].detach().cpu()*(out>0.35)).numpy()\n    answers = []\n    for i in ans:\n        if(i.sum()>0):\n            tmp = []\n            for x in i:\n                if x==0:\n                    word = tokenizer.decode(tmp)\n                    if len(word.split())>2:\n                        answers.append(clean_text(word))\n                    tmp = []\n                else:\n                    tmp.append(x)\n            word = tokenizer.decode(tmp)\n            if len(word.split())>2:\n                answers.append(clean_text(word))\n    if len(answers)==0:\n        return []\n#     c = []\n#     for ans in answers:\n#         c.append(text.count(ans))\n#     answers = [answers[i] for i in np.argsort(c)[::-1][:3]]\n    return answers    ","metadata":{"execution":{"iopub.status.busy":"2021-06-11T04:08:33.103213Z","iopub.execute_input":"2021-06-11T04:08:33.103566Z","iopub.status.idle":"2021-06-11T04:08:33.114552Z","shell.execute_reply.started":"2021-06-11T04:08:33.103515Z","shell.execute_reply":"2021-06-11T04:08:33.113754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dsets = []\nids = []\ndset_comb = []\nfor i in tqdm(range(len(submission))):\n        text = json.load(open(os.path.join(test_path,submission.iloc[i,0]+\".json\")))\n        sec = []\n        for x in text:\n            sec.append(x['section_title'])\n            sec.append(\" \")    \n            sec.append(x['text'])    \n        entire = \"\".join(sec)\n        pred1 = find_dataset_rule(entire)\n        pred2 = find_dataset(clean_text(entire))\n        pred3 = []\n        try:\n            pred3 = find_dataset_ann(entire)\n        except:\n            torch.cuda.empty_cache()\n            print(\"error\")\n        pred = []\n        for p in pred1: pred.append(p)\n        for p in pred2: pred.append(p)\n        for p in pred3: pred.append(p)\n        pred = set(pred)\n        dset_comb.append(pred)\n        for p in pred:\n            dsets.append(pred)\n            ids.append(i)\ndatasets = pd.DataFrame()\ndatasets['ids']=ids\ndatasets['dsets'] = dsets","metadata":{"execution":{"iopub.status.busy":"2021-06-11T04:08:33.116176Z","iopub.execute_input":"2021-06-11T04:08:33.116758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,spl in enumerate(dset_comb):\n    n_spl = []\n    for s in spl:\n        if len(datasets[(datasets.ids!=i)&(datasets.dsets==s)])>0:\n            n_spl.append(s)\n    submission.loc[i,\"PredictionString\"] = \"|\".join(n_spl)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:32:18.333009Z","iopub.execute_input":"2021-06-10T10:32:18.333385Z","iopub.status.idle":"2021-06-10T10:32:18.361878Z","shell.execute_reply.started":"2021-06-10T10:32:18.333351Z","shell.execute_reply":"2021-06-10T10:32:18.361041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","metadata":{"papermill":{"duration":0.034632,"end_time":"2021-06-02T03:57:57.779228","exception":false,"start_time":"2021-06-02T03:57:57.744596","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-10T10:32:25.747518Z","iopub.execute_input":"2021-06-10T10:32:25.747898Z","iopub.status.idle":"2021-06-10T10:32:25.762153Z","shell.execute_reply.started":"2021-06-10T10:32:25.747868Z","shell.execute_reply":"2021-06-10T10:32:25.761323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-06-10T10:32:34.341976Z","iopub.execute_input":"2021-06-10T10:32:34.342307Z","iopub.status.idle":"2021-06-10T10:32:34.351604Z","shell.execute_reply.started":"2021-06-10T10:32:34.342276Z","shell.execute_reply":"2021-06-10T10:32:34.350627Z"},"trusted":true},"execution_count":null,"outputs":[]}]}