{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\ntrain_files = glob.glob(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train/*.json\")\ntest_files = glob.glob(\"/kaggle/input/coleridgeinitiative-show-us-the-data/test/*.json\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\n# Generate the training publications dataframe\ndf_train_publications = pd.DataFrame()\n\nfor train_file in tqdm(train_files):\n    file_data = pd.read_json(train_file)\n    file_data.insert(0,'pub_id', train_file.split('/')[-1].split('.')[0].replace('train\\\\', ''))\n    df_train_publications = pd.concat([df_train_publications, file_data])\n\ndf_train_publications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the testing publications dataframe\ndf_test_publications = pd.DataFrame()\n\nfor test_file in tqdm(test_files):\n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'pub_id', test_file.split('/')[-1].split('.')[0].replace('test\\\\', ''))\n    df_test_publications = pd.concat([df_test_publications, file_data])\n\ndf_test_publications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def concat(column):\n    res = ' '\n    for st in column:\n        if type(st) == str:\n            res += st\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = df_train_publications.groupby('pub_id')['text'].apply(concat).reset_index()\n# train = df_train_publications\ntrain.loc[train['pub_id'].isin(train_csv['Id']), 'cleaned_label'] = train_csv.loc[train_csv['Id'].isin(train['pub_id']),'cleaned_label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.cleaned_label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train = pd.DataFrame([])\ntrain = train.set_index(pd.Index(range(len(train))))\nfor idx in train.index:\n    if train.loc[idx, 'text'].find(train.loc[idx, 'cleaned_label']) >= 0:\n        new_train = pd.concat([new_train, train.loc[idx:idx+1, :].drop(index=idx+1)], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(new_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test = df_test_publications.groupby('pub_id')['text'].apply(concat).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_size = 200\nsentences = new_train['text']\nlabels = new_train['cleaned_label']\n\ntrain_contexts = sentences[training_size:]\n\ntrain_answers = labels[training_size:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import DistilBertForQuestionAnswering\nmodel = DistilBertForQuestionAnswering.from_pretrained(\"distilbert-base-uncased\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_answers = train_answers.to_list()\ntrain_contexts = train_contexts.to_list()\n\ntrain_answers = [{\"text\":answ} for answ in train_answers]\n\n\ndef add_end_idx(answers, contexts):\n    i = 0\n\n    for answer, context in zip(answers, contexts):\n        gold_text = answer['text']\n        start_idx = context.find(answer['text'])\n        end_idx = start_idx + len(gold_text)\n\n        # sometimes squad answers are off by a character or two – fix this\n        if context[start_idx:end_idx] == gold_text:\n            answers[i]['answer_start'] = start_idx\n            answers[i]['answer_end'] = end_idx\n        elif context[start_idx-1:end_idx-1] == gold_text:\n            answers[i]['answer_start'] = start_idx - 1\n            answers[i]['answer_end'] = end_idx - 1     # When the gold label is off by one character\n        elif context[start_idx-2:end_idx-2] == gold_text:\n            answers[i]['answer_start'] = start_idx - 2\n            answers[i]['answer_end'] = end_idx - 2     # When the gold label is off by two characters\n        else:\n            answers[i]['answer_start'] = start_idx + 1 \n        i+=1\n\nadd_end_idx(train_answers, train_contexts)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_answers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import DistilBertTokenizerFast\ntokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n\ntrain_questions = ['What is the dataset?']*len(train_contexts)\n\ntrain_encodings = tokenizer(train_contexts, train_questions, truncation=True, padding=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_token_positions(encodings, answers):\n    start_positions = []\n    end_positions = []\n    for i in range(len(answers)):\n        start_positions.append(encodings.char_to_token(i, answers[i]['answer_start']))\n        end_positions.append(encodings.char_to_token(i, answers[i]['answer_end'] - 1))\n\n        # if start position is None, the answer passage has been truncated\n        if start_positions[-1] is None:\n            start_positions[-1] = tokenizer.model_max_length\n        if end_positions[-1] is None:\n            end_positions[-1] = tokenizer.model_max_length\n\n    encodings.update({'start_positions': start_positions, 'end_positions': end_positions})\n\nadd_token_positions(train_encodings, train_answers)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\n\nclass SquadDataset(torch.utils.data.Dataset):\n    def __init__(self, encodings):\n        self.encodings = encodings\n\n    def __getitem__(self, idx):\n        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n\n    def __len__(self):\n        return len(self.encodings.input_ids)\n\ntrain_dataset = SquadDataset(train_encodings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import DistilBertForQuestionAnswering\nmodel = DistilBertForQuestionAnswering.from_pretrained(\"distilbert-base-uncased\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader\nfrom transformers import AdamW\n\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\nmodel.to(device)\nmodel.train()\n\ntrain_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n\noptim = AdamW(model.parameters(), lr=5e-5)\n\nfor epoch in range(3):\n    for batch in tqdm(train_loader):\n        optim.zero_grad()\n        input_ids = batch['input_ids'].to(device)\n        attention_mask = batch['attention_mask'].to(device)\n        start_positions = batch['start_positions'].to(device)\n        end_positions = batch['end_positions'].to(device)\n        outputs = model(input_ids, attention_mask=attention_mask, start_positions=start_positions, end_positions=end_positions)\n        loss = outputs[0]\n        loss.backward()\n        optim.step()\n\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}