{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-23T06:58:48.707211Z","iopub.execute_input":"2021-05-23T06:58:48.707807Z","iopub.status.idle":"2021-05-23T06:59:03.312486Z","shell.execute_reply.started":"2021-05-23T06:58:48.707681Z","shell.execute_reply":"2021-05-23T06:59:03.311344Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find the dataset with model @ - https://www.kaggle.com/ash1706/ner-model","metadata":{}},{"cell_type":"markdown","source":"This notebook trains a NER BERT BASE model \nCredits to https://www.kaggle.com/tungmphung/pytorch-bert-for-named-entity-recognition fro such a amazing notebook\nand all the owners of the dataset i am using\nalso a great NER data processing notebook - https://www.kaggle.com/devashishprasad/bio-labeled-dataset","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-05-23T06:59:03.316047Z","iopub.execute_input":"2021-05-23T06:59:03.316363Z","iopub.status.idle":"2021-05-23T06:59:40.177021Z","shell.execute_reply.started":"2021-05-23T06:59:03.316332Z","shell.execute_reply":"2021-05-23T06:59:40.175790Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Lets load the NER labeled data set","metadata":{}},{"cell_type":"code","source":"import pickle\nimport pandas as pd\nimport numpy as np\n\nX = []\ny = []\n\nwith open('../input/test-labels-ner/sentences.pkl', 'rb') as f:\n    X = pickle.load(f)\n    \nwith open('../input/test-labels-ner/bio_labels.pkl', 'rb') as f:\n    y = pickle.load(f)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-23T06:59:40.179668Z","iopub.execute_input":"2021-05-23T06:59:40.180130Z","iopub.status.idle":"2021-05-23T07:00:45.310375Z","shell.execute_reply.started":"2021-05-23T06:59:40.180080Z","shell.execute_reply":"2021-05-23T07:00:45.308839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:45.313299Z","iopub.execute_input":"2021-05-23T07:00:45.313775Z","iopub.status.idle":"2021-05-23T07:00:46.156835Z","shell.execute_reply.started":"2021-05-23T07:00:45.313733Z","shell.execute_reply":"2021-05-23T07:00:46.155715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/coleridge-packages/my_seqeval.py ./","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:46.158464Z","iopub.execute_input":"2021-05-23T07:00:46.158924Z","iopub.status.idle":"2021-05-23T07:00:47.143360Z","shell.execute_reply.started":"2021-05-23T07:00:46.158881Z","shell.execute_reply":"2021-05-23T07:00:47.142129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nMAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:47.147489Z","iopub.execute_input":"2021-05-23T07:00:47.147812Z","iopub.status.idle":"2021-05-23T07:00:47.153915Z","shell.execute_reply.started":"2021-05-23T07:00:47.147779Z","shell.execute_reply":"2021-05-23T07:00:47.152619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\nprint(f'No. raw training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:47.155839Z","iopub.execute_input":"2021-05-23T07:00:47.156573Z","iopub.status.idle":"2021-05-23T07:00:47.282115Z","shell.execute_reply.started":"2021-05-23T07:00:47.156528Z","shell.execute_reply":"2021-05-23T07:00:47.280860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:47.285802Z","iopub.execute_input":"2021-05-23T07:00:47.286267Z","iopub.status.idle":"2021-05-23T07:00:47.745859Z","shell.execute_reply.started":"2021-05-23T07:00:47.286224Z","shell.execute_reply":"2021-05-23T07:00:47.744649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# papers = {}\n# for paper_id in train['Id'].unique():\n#     with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n#         paper = json.load(f)\n#         papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:47.747986Z","iopub.execute_input":"2021-05-23T07:00:47.748699Z","iopub.status.idle":"2021-05-23T07:00:47.753620Z","shell.execute_reply.started":"2021-05-23T07:00:47.748651Z","shell.execute_reply":"2021-05-23T07:00:47.752319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c=0\nco = 0","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:47.755268Z","iopub.execute_input":"2021-05-23T07:00:47.756036Z","iopub.status.idle":"2021-05-23T07:00:47.765844Z","shell.execute_reply.started":"2021-05-23T07:00:47.755990Z","shell.execute_reply":"2021-05-23T07:00:47.764597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"you can play around with how much data you want to use i would suggest to use all of it but then again i was testing this approach so using all of it didnt make sense as it would require a lot of time and resources","metadata":{}},{"cell_type":"markdown","source":"# use version 8 for a bigger sample","metadata":{}},{"cell_type":"code","source":"with open('train_ner.json', 'w') as f:\n    for i in range(len(X)):\n        words = X[i]\n        nes = y[i]\n#         row_json = {'tokens' : words, 'tags' : nes}\n#         co = co+1\n#         json.dump(row_json, f)\n#         f.write('\\n')\n        if('B' in nes):\n            row_json = {'tokens' : words, 'tags' : nes}\n            co = co+1\n            json.dump(row_json, f)\n            f.write('\\n')\n#         else:\n#             c = c+1\n#             if(c>100):\n#                 c=0\n#                 co=co+1\n#                 row_json = {'tokens' : words, 'tags' : nes}\n#                 json.dump(row_json, f)\n#                 f.write('\\n')\n                \n                ","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:47.767675Z","iopub.execute_input":"2021-05-23T07:00:47.768312Z","iopub.status.idle":"2021-05-23T07:00:55.155095Z","shell.execute_reply.started":"2021-05-23T07:00:47.768268Z","shell.execute_reply":"2021-05-23T07:00:55.153946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ndel y\n# will run into memory issues otherwise","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:00:55.157159Z","iopub.execute_input":"2021-05-23T07:00:55.157635Z","iopub.status.idle":"2021-05-23T07:01:00.863644Z","shell.execute_reply.started":"2021-05-23T07:00:55.157576Z","shell.execute_reply":"2021-05-23T07:01:00.862279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(co)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:01:00.865954Z","iopub.execute_input":"2021-05-23T07:01:00.866582Z","iopub.status.idle":"2021-05-23T07:01:00.876659Z","shell.execute_reply.started":"2021-05-23T07:01:00.866535Z","shell.execute_reply":"2021-05-23T07:01:00.874994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(X)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:01:00.878278Z","iopub.execute_input":"2021-05-23T07:01:00.878984Z","iopub.status.idle":"2021-05-23T07:01:00.890206Z","shell.execute_reply.started":"2021-05-23T07:01:00.878937Z","shell.execute_reply":"2021-05-23T07:01:00.888750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade git+https://github.com/intake/filesystem_spec\n# comment out if you want to run without internet","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:01:00.891829Z","iopub.execute_input":"2021-05-23T07:01:00.892631Z","iopub.status.idle":"2021-05-23T07:01:22.229152Z","shell.execute_reply.started":"2021-05-23T07:01:00.892585Z","shell.execute_reply":"2021-05-23T07:01:22.227897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training begins","metadata":{}},{"cell_type":"code","source":"!python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n--model_name_or_path 'bert-base-cased' \\\n--train_file './train_ner.json' \\\n--validation_file './train_ner.json' \\\n--num_train_epochs 1  \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--save_steps 15000 \\\n--output_dir './output' \\\n--report_to 'none' \\\n--seed 123 \\\n--do_train","metadata":{"execution":{"iopub.status.busy":"2021-05-23T07:02:52.295399Z","iopub.execute_input":"2021-05-23T07:02:52.298555Z","iopub.status.idle":"2021-05-23T07:39:22.033569Z","shell.execute_reply.started":"2021-05-23T07:02:52.298502Z","shell.execute_reply":"2021-05-23T07:39:22.032327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}