{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport re\n\nimport glob, json, os\n\nimport torch \n\nfrom scipy.spatial.distance import euclidean, cosine\n\nfrom transformers import BertConfig, DebertaTokenizer, DebertaModel, RobertaTokenizer, RobertaModel, MPNetTokenizer, MPNetModel, XLMTokenizer, XLMModel, BertTokenizer, BertModel","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-17T16:12:53.727191Z","iopub.execute_input":"2021-06-17T16:12:53.728028Z","iopub.status.idle":"2021-06-17T16:12:53.739710Z","shell.execute_reply.started":"2021-06-17T16:12:53.727953Z","shell.execute_reply":"2021-06-17T16:12:53.738499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Train, Test & Pretrained Models","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:12:53.744781Z","iopub.execute_input":"2021-06-17T16:12:53.745968Z","iopub.status.idle":"2021-06-17T16:12:53.759016Z","shell.execute_reply.started":"2021-06-17T16:12:53.745926Z","shell.execute_reply":"2021-06-17T16:12:53.757825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\n\nprint(df.shape)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:12:53.761086Z","iopub.execute_input":"2021-06-17T16:12:53.762072Z","iopub.status.idle":"2021-06-17T16:12:53.887661Z","shell.execute_reply.started":"2021-06-17T16:12:53.762028Z","shell.execute_reply":"2021-06-17T16:12:53.886470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vals = pd.concat([df.dataset_title, df.dataset_label, df.cleaned_label])\nvals = np.unique([clean_text(t) for t in vals])\nprint(len(vals))\nvals[:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:12:53.890174Z","iopub.execute_input":"2021-06-17T16:12:53.890643Z","iopub.status.idle":"2021-06-17T16:12:54.270414Z","shell.execute_reply.started":"2021-06-17T16:12:53.890597Z","shell.execute_reply":"2021-06-17T16:12:54.268987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_vals = pd.DataFrame({\"values\":vals, \"ntokens\":[len(v.split()) for v in vals]})\nprint(df_vals.shape)\ndf_vals.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:12:54.273122Z","iopub.execute_input":"2021-06-17T16:12:54.273725Z","iopub.status.idle":"2021-06-17T16:12:54.289819Z","shell.execute_reply.started":"2021-06-17T16:12:54.273675Z","shell.execute_reply":"2021-06-17T16:12:54.288487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_vals.describe()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:12:54.291590Z","iopub.execute_input":"2021-06-17T16:12:54.292530Z","iopub.status.idle":"2021-06-17T16:12:54.315232Z","shell.execute_reply.started":"2021-06-17T16:12:54.292470Z","shell.execute_reply":"2021-06-17T16:12:54.314138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use gpu if available\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n#deberta\ntokenizer_d = DebertaTokenizer.from_pretrained(\"../input/deberta2/v2/deberta-base-tok\")\nmodel_d = DebertaModel.from_pretrained(\"../input/deberta2/v2/deberta-base-model\").to(device)\n\n#bert base max\ntokenizer_bbmax = BertTokenizer.from_pretrained(\"../input/embedders/bert-base-nli-max-tokens-tok\")\nmodel_bbmax = BertModel.from_pretrained(\"../input/embedders/bert-base-nli-max-tokens-model\").to(device)\n\n#bert base mean\ntokenizer_bbmean = BertTokenizer.from_pretrained(\"../input/embedders/bert-base-nli-mean-tokens-tok\")\nmodel_bbmean = BertModel.from_pretrained(\"../input/embedders/bert-base-nli-mean-tokens-model\").to(device)\n\n#paraphrase mini\ntokenizer_pmini = BertTokenizer.from_pretrained(\"../input/embedders/paraphrase-MiniLM-tok\")\nmodel_pmini = BertModel.from_pretrained(\"../input/embedders/paraphrase-MiniLM-model\").to(device)\n\n#paraphrase distilroberta\ntokenizer_pd = RobertaTokenizer.from_pretrained(\"../input/embedders/paraphrase-distilroberta-tok\")\nmodel_pd = RobertaModel.from_pretrained(\"../input/embedders/paraphrase-distilroberta-model\").to(device)\n\n#paraphrase mpnet\ntokenizer_pm = MPNetTokenizer.from_pretrained(\"../input/pmpnet/paraphrase-mpnet-tok\")\nmodel_pm = MPNetModel.from_pretrained(\"../input/pmpnet/paraphrase-mpnet-model\").to(device)\n\n#roberta\ntokenizer_r = RobertaTokenizer.from_pretrained(\"../input/embedders/roberta-base-tok\")\nmodel_r = RobertaModel.from_pretrained(\"../input/embedders/roberta-base-model\").to(device)\n\nprint(model_d.config, model_bbmax.config, model_bbmean.config, model_pmini.config, model_pd.config, model_pm.config, model_r.config)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:12:54.317358Z","iopub.execute_input":"2021-06-17T16:12:54.318033Z","iopub.status.idle":"2021-06-17T16:13:13.197930Z","shell.execute_reply.started":"2021-06-17T16:12:54.317987Z","shell.execute_reply":"2021-06-17T16:13:13.196798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model_d.embeddings.word_embeddings, model_bbmax.embeddings.word_embeddings, model_bbmean.embeddings.word_embeddings, \n      model_pmini.embeddings.word_embeddings, model_pd.embeddings.word_embeddings, model_pm.embeddings.word_embeddings, \n      model_r.embeddings.word_embeddings)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:13.202343Z","iopub.execute_input":"2021-06-17T16:13:13.202667Z","iopub.status.idle":"2021-06-17T16:13:13.210004Z","shell.execute_reply.started":"2021-06-17T16:13:13.202637Z","shell.execute_reply":"2021-06-17T16:13:13.208779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model_d.device, model_bbmax.device, model_bbmean.device, model_pmini.device, model_pd.device, model_pm.device, model_r.device)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:13.212350Z","iopub.execute_input":"2021-06-17T16:13:13.213023Z","iopub.status.idle":"2021-06-17T16:13:13.228853Z","shell.execute_reply.started":"2021-06-17T16:13:13.212969Z","shell.execute_reply":"2021-06-17T16:13:13.227393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load(json_type=str):\n    base_path = f\"../input/coleridgeinitiative-show-us-the-data/{json_type}\"\n        \n    texts = []\n\n    for j in os.listdir(base_path):\n        print(j)\n        texts.append(pd.read_json(os.path.join(base_path, j)))\n    \n    df_test = pd.DataFrame()\n    \n    df_test[\"Id\"] = pd.Series([j.strip(\".json\") for j in os.listdir(base_path)], name=\"Id\")\n    df_test[\"text\"] = pd.Series([\" \".join(t[\"text\"]) for t in texts], name=\"text\")\n    df_test.text = df_test.text.apply(lambda row: clean_text(row))\n    \n    return df_test","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:13.230724Z","iopub.execute_input":"2021-06-17T16:13:13.231482Z","iopub.status.idle":"2021-06-17T16:13:13.242475Z","shell.execute_reply.started":"2021-06-17T16:13:13.231434Z","shell.execute_reply":"2021-06-17T16:13:13.241231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = load(\"test\")\nprint(df_test.shape)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:13.243963Z","iopub.execute_input":"2021-06-17T16:13:13.244750Z","iopub.status.idle":"2021-06-17T16:13:13.331470Z","shell.execute_reply.started":"2021-06-17T16:13:13.244705Z","shell.execute_reply":"2021-06-17T16:13:13.330306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.text[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:13.333036Z","iopub.execute_input":"2021-06-17T16:13:13.333456Z","iopub.status.idle":"2021-06-17T16:13:13.343946Z","shell.execute_reply.started":"2021-06-17T16:13:13.333394Z","shell.execute_reply":"2021-06-17T16:13:13.342484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dictionary to create tokens of length == train vals from json test","metadata":{}},{"cell_type":"code","source":"import time\n\nstart = time.time()\n\nempty = {k:{} for k in range(len(df_test.text))}\n\nfor l, t in zip(range(len(empty)), df_test.text):\n    splitted = np.array(t.split())\n    vals_d = {v:[] for v in vals}\n    for v in vals:\n        n = len(v.split())\n        for i in range(len(splitted)):\n            vals_d[v].append(splitted[i:n+i])\n    empty[l] = vals_d\n\nend = time.time()\nprint(end - start)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:13.346161Z","iopub.execute_input":"2021-06-17T16:13:13.346730Z","iopub.status.idle":"2021-06-17T16:13:19.753934Z","shell.execute_reply.started":"2021-06-17T16:13:13.346671Z","shell.execute_reply":"2021-06-17T16:13:19.752626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(empty), type(empty), \"\\n\")\nprint(len(empty[1].keys()), empty[1].keys())","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:19.755689Z","iopub.execute_input":"2021-06-17T16:13:19.756405Z","iopub.status.idle":"2021-06-17T16:13:19.765863Z","shell.execute_reply.started":"2021-06-17T16:13:19.756322Z","shell.execute_reply":"2021-06-17T16:13:19.764622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"empty[0].keys() == empty[2].keys()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:19.767304Z","iopub.execute_input":"2021-06-17T16:13:19.767824Z","iopub.status.idle":"2021-06-17T16:13:19.779147Z","shell.execute_reply.started":"2021-06-17T16:13:19.767776Z","shell.execute_reply":"2021-06-17T16:13:19.777658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(vals) == list(empty[1].keys())","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:19.781049Z","iopub.execute_input":"2021-06-17T16:13:19.781698Z","iopub.status.idle":"2021-06-17T16:13:19.794098Z","shell.execute_reply.started":"2021-06-17T16:13:19.781654Z","shell.execute_reply":"2021-06-17T16:13:19.792483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_v = pd.DataFrame([\" \".join(t) for t in empty[2][\"trends in international mathematics and science study\"]], columns=[\"val\"])\nprint(df_v.shape)\ndf_v[df_v.val==(\"trends in international mathematics and science study\")]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:19.795776Z","iopub.execute_input":"2021-06-17T16:13:19.796525Z","iopub.status.idle":"2021-06-17T16:13:19.977548Z","shell.execute_reply.started":"2021-06-17T16:13:19.796466Z","shell.execute_reply":"2021-06-17T16:13:19.976054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models Test","metadata":{}},{"cell_type":"code","source":"#deberta: tokenizer_d, model_d\n\n#bert-base-max: tokenizer_bbmax, model_bbmax\n\n#bert-base-mean: tokenizer_bbmean, model_bbmean\n\n#paraphrase-mini: tokenizer_pmini, model_pmini\n\n#paraphrase-distilroberta: tokenizer_pd, model_pd\n\n#paraphrase-mpnet: tokenizer_pm, model_pm\n\n#roberta: tokenizer_r, model_r","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:19.979643Z","iopub.execute_input":"2021-06-17T16:13:19.980142Z","iopub.status.idle":"2021-06-17T16:13:19.985507Z","shell.execute_reply.started":"2021-06-17T16:13:19.980072Z","shell.execute_reply":"2021-06-17T16:13:19.984023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### paraphrase-mini: tokenizer_pmini, model_pmini","metadata":{}},{"cell_type":"code","source":"encoded_v = tokenizer_pmini.encode_plus((\"trends in international mathematics and science study\").split(), padding=True, truncation=True, add_special_tokens=False, return_attention_mask=True, return_tensors=\"pt\")\nwith torch.no_grad():\n    output_v = model_pmini(**encoded_v.to(device))\n#output_v = model_t(**encoded_v)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:19.987432Z","iopub.execute_input":"2021-06-17T16:13:19.988306Z","iopub.status.idle":"2021-06-17T16:13:20.018030Z","shell.execute_reply.started":"2021-06-17T16:13:19.988259Z","shell.execute_reply":"2021-06-17T16:13:20.016953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use gpu is available\n#device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\ns = time.time()\nencoded_ = [tokenizer_pmini.encode_plus(\" \".join(t), padding=False, truncation=True, add_special_tokens=False, return_attention_mask=True, return_tensors=\"pt\") for t in empty[2][\"trends in international mathematics and science study\"]]\nwith torch.no_grad():\n    output_ = [model_pmini(**e.to(device)) for e in encoded_]\n#output_ = model_t(**encoded_)\ne = time.time()\nprint(e-s)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:13:20.022273Z","iopub.execute_input":"2021-06-17T16:13:20.022609Z","iopub.status.idle":"2021-06-17T16:18:10.090120Z","shell.execute_reply.started":"2021-06-17T16:13:20.022579Z","shell.execute_reply":"2021-06-17T16:18:10.088749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[\" \".join(t).encode() for t in empty[2][\"trends in international mathematics and science study\"][:2]]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:18:10.104954Z","iopub.execute_input":"2021-06-17T16:18:10.105276Z","iopub.status.idle":"2021-06-17T16:18:10.123941Z","shell.execute_reply.started":"2021-06-17T16:18:10.105245Z","shell.execute_reply":"2021-06-17T16:18:10.122150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_v[0].cpu().detach().numpy().shape, output_[1][0].cpu().detach().numpy().shape, len([\" \".join(t) for t in empty[2][\"trends in international mathematics and science study\"]])","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:18:10.126076Z","iopub.execute_input":"2021-06-17T16:18:10.126613Z","iopub.status.idle":"2021-06-17T16:18:10.295859Z","shell.execute_reply.started":"2021-06-17T16:18:10.126557Z","shell.execute_reply":"2021-06-17T16:18:10.294771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(output_)):\n    #output_[i][0].detach().numpy().flatten()\n    d = cosine(np.average(output_v[0].cpu().detach().numpy(), axis=1), np.average(output_[i][0].cpu().detach().numpy(), axis=1))\n    if d < 0.05:\n        print(d, empty[2][\"trends in international mathematics and science study\"][i])","metadata":{"execution":{"iopub.status.busy":"2021-06-17T16:18:10.299371Z","iopub.execute_input":"2021-06-17T16:18:10.299775Z","iopub.status.idle":"2021-06-17T16:18:19.778284Z","shell.execute_reply.started":"2021-06-17T16:18:10.299743Z","shell.execute_reply":"2021-06-17T16:18:19.777110Z"},"trusted":true},"execution_count":null,"outputs":[]}]}