{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Finetune SpaCy NER","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport random\nimport numpy as np\nimport pandas as pd\nfrom time import time\nimport warnings\n\nimport spacy\nfrom spacy.util import minibatch, compounding","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-07T09:51:14.079767Z","iopub.execute_input":"2021-06-07T09:51:14.080286Z","iopub.status.idle":"2021-06-07T09:51:14.782376Z","shell.execute_reply.started":"2021-06-07T09:51:14.080176Z","shell.execute_reply":"2021-06-07T09:51:14.781559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load NLP and Data","metadata":{}},{"cell_type":"code","source":"nlp = spacy.load(\"../input/coleridge-ner-chain-v02-c03/coleridge_ner\")\nwith open(\"../input/coleridgecuratednerlabels002/train-1234.json\") as rf:\n train_dict = json.load(rf)\ntrain_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-06T10:06:13.696458Z","iopub.execute_input":"2021-06-06T10:06:13.696836Z","iopub.status.idle":"2021-06-06T10:06:36.110788Z","shell.execute_reply.started":"2021-06-06T10:06:13.696808Z","shell.execute_reply":"2021-06-06T10:06:36.109961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sampling Rates","metadata":{}},{"cell_type":"code","source":"dsets = train_df['dataset_label'].value_counts().to_frame().reset_index()\ndsets.columns = ['dataset_label', \"N\"]\ndsets.loc[:, \"log_N\"] = np.log(dsets[\"N\"])\ndsets.loc[:, \"sample_rate\"] = 0.99 - dsets[\"log_N\"]/9.5\n\ntrain_id = pd.merge(train_df[[\"Id\", \"dataset_label\"]], dsets[[\"dataset_label\", \"sample_rate\"]], on=\"dataset_label\")\ntrain_id = train_id.groupby(\"Id\")[\"sample_rate\"].mean().reset_index()\ntrain_id.loc[:, \"sample_rate\"] = np.round(train_id[\"sample_rate\"], 2)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T10:32:55.466346Z","iopub.execute_input":"2021-06-06T10:32:55.466686Z","iopub.status.idle":"2021-06-06T10:32:55.584842Z","shell.execute_reply.started":"2021-06-06T10:32:55.466659Z","shell.execute_reply":"2021-06-06T10:32:55.583916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tuning Config","metadata":{}},{"cell_type":"code","source":"N_it = 20\nb0 = 24\nb1 = 192\nr = 1.0025\ndrop_rate = 0.11\nseed = 121","metadata":{"execution":{"iopub.status.busy":"2021-06-06T10:31:41.242157Z","iopub.execute_input":"2021-06-06T10:31:41.242522Z","iopub.status.idle":"2021-06-06T10:31:41.247138Z","shell.execute_reply.started":"2021-06-06T10:31:41.242492Z","shell.execute_reply":"2021-06-06T10:31:41.246358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Selection","metadata":{}},{"cell_type":"code","source":"np.random.seed(seed)\ntrain_set = []\nfor sr in np.unique(train_id.sample_rate):\n \n id_list = train_id.query(\"sample_rate == @sr\")[\"Id\"].tolist()\n train_data = []\n for k, v in train_dict.items():\n if k in id_list:\n for x in v:\n train_data.append((x['example'][0], x['example'][1]))\n \n N_eg = int(sr * len(train_data))\n train_idx = np.random.choice(range(len(train_data)), size=min(len(train_data), N_eg), replace=False)\n train_sub = [train_data[i] for i in train_idx]\n train_set += train_sub\n\nlen(train_set)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T10:32:57.321081Z","iopub.execute_input":"2021-06-06T10:32:57.32146Z","iopub.status.idle":"2021-06-06T10:33:03.742344Z","shell.execute_reply.started":"2021-06-06T10:32:57.321426Z","shell.execute_reply":"2021-06-06T10:33:03.741417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training Prep","metadata":{}},{"cell_type":"code","source":"ner = nlp.get_pipe(\"ner\")\nfor _, annotations in train_set:\n for ent in annotations.get(\"entities\"):\n ner.add_label(ent[2])\nunaffected_pipes = [pipe for pipe in nlp.pipe_names if pipe != \"ner\"]","metadata":{"execution":{"iopub.status.busy":"2021-06-03T13:11:39.155885Z","iopub.execute_input":"2021-06-03T13:11:39.156532Z","iopub.status.idle":"2021-06-03T13:11:41.557759Z","shell.execute_reply.started":"2021-06-03T13:11:39.156482Z","shell.execute_reply":"2021-06-03T13:11:41.556932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fine-tuning","metadata":{}},{"cell_type":"code","source":"with nlp.disable_pipes(*unaffected_pipes):\n \n # disable warnings\n warnings.filterwarnings(\"ignore\", category=UserWarning, module='spacy') \n \n # Training for N iterations\n for it_num in range(N_it):\n\n # shuffling examples before every iteration\n t0 = time()\n random.shuffle(train_set)\n losses = {}\n\n # batch up the examples using spaCy's minibatch\n batches = minibatch(train_set, size=compounding(b0, b1, r))\n for batch in batches:\n texts, annotations = zip(*batch)\n nlp.update(texts, annotations, drop=drop_rate, losses=losses)\n \n # loss progress\n print(\"Round %d loss: %.2f (%.1f seconds)\"%(it_num, losses['ner'], time() - t0))\n\n# save to disk \nnlp.to_disk(\"coleridge_ner\")","metadata":{"execution":{"iopub.status.busy":"2021-06-03T13:11:52.053824Z","iopub.execute_input":"2021-06-03T13:11:52.054452Z","iopub.status.idle":"2021-06-03T13:12:34.709193Z","shell.execute_reply.started":"2021-06-03T13:11:52.054416Z","shell.execute_reply":"2021-06-03T13:12:34.706074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***\n
","metadata":{}}]}