{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook shows how to fine-tune a BERT model (from huggingface) for our dataset recognition task.\n\nNote that internet is needed during the training phase (for downloading the bert-base-cased model). Internet can be turned off during prediction.","metadata":{}},{"cell_type":"markdown","source":"## Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-11T07:37:01.902497Z","iopub.execute_input":"2021-06-11T07:37:01.902841Z","iopub.status.idle":"2021-06-11T07:37:29.610324Z","shell.execute_reply.started":"2021-06-11T07:37:01.902765Z","shell.execute_reply":"2021-06-11T07:37:29.609438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:37:29.613549Z","iopub.execute_input":"2021-06-11T07:37:29.613923Z","iopub.status.idle":"2021-06-11T07:37:30.410075Z","shell.execute_reply.started":"2021-06-11T07:37:29.613873Z","shell.execute_reply":"2021-06-11T07:37:30.409292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:37:36.86792Z","iopub.execute_input":"2021-06-11T07:37:36.868275Z","iopub.status.idle":"2021-06-11T07:37:37.505199Z","shell.execute_reply.started":"2021-06-11T07:37:36.86822Z","shell.execute_reply":"2021-06-11T07:37:37.504255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyper-parameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nMAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:37:39.035977Z","iopub.execute_input":"2021-06-11T07:37:39.036316Z","iopub.status.idle":"2021-06-11T07:37:39.041067Z","shell.execute_reply.started":"2021-06-11T07:37:39.036283Z","shell.execute_reply":"2021-06-11T07:37:39.040004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\nprint(f'No. raw training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:37:40.791882Z","iopub.execute_input":"2021-06-11T07:37:40.792334Z","iopub.status.idle":"2021-06-11T07:37:40.930578Z","shell.execute_reply.started":"2021-06-11T07:37:40.792291Z","shell.execute_reply":"2021-06-11T07:37:40.929707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:38:06.104126Z","iopub.execute_input":"2021-06-11T07:38:06.104484Z","iopub.status.idle":"2021-06-11T07:38:06.455078Z","shell.execute_reply.started":"2021-06-11T07:38:06.104452Z","shell.execute_reply":"2021-06-11T07:38:06.454115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# upsampling code","metadata":{}},{"cell_type":"code","source":"uncommend this part to upsample the low-frequency labels","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cleaned_counts = train[\"cleaned_label\"].value_counts()\n# values = cleaned_counts.keys().tolist()\n# counts = cleaned_counts.tolist()\n\n# to_duplicate=[]\n# nr_dumplicate=5 # change this to the minimal number of occurances you want each entry to have \n","metadata":{"execution":{"iopub.status.busy":"2021-06-10T12:24:23.132132Z","iopub.execute_input":"2021-06-10T12:24:23.134705Z","iopub.status.idle":"2021-06-10T12:24:23.14023Z","shell.execute_reply.started":"2021-06-10T12:24:23.134666Z","shell.execute_reply":"2021-06-10T12:24:23.139234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train[\"cleaned_label\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T12:24:23.143448Z","iopub.execute_input":"2021-06-10T12:24:23.144684Z","iopub.status.idle":"2021-06-10T12:24:23.151971Z","shell.execute_reply.started":"2021-06-10T12:24:23.144647Z","shell.execute_reply":"2021-06-10T12:24:23.15092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(0,nr_dumplicate):\n#     print(i)\n# #     extract the cleaned label if it occurs less than 5 times\n#     for i,counts in enumerate(cleaned_counts):\n#         if cleaned_counts[i]<nr_dumplicate:\n#             to_duplicate.append(values[i])\n\n#     list_df = [[] for i in range(len(to_duplicate))]\n#     # makes a dataframe where it adds the row of data that contains the one we want to duplicate\n#     for i,value in enumerate(to_duplicate):\n#         list_df[i] = (train.loc[train[\"cleaned_label\"]== to_duplicate[i]])\n\n#     duplicated_train = pd.concat(list_df)\n#     frames = [train, duplicated_train]\n#     new_train = pd.concat(frames)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T12:24:23.153294Z","iopub.execute_input":"2021-06-10T12:24:23.154366Z","iopub.status.idle":"2021-06-10T12:24:23.161265Z","shell.execute_reply.started":"2021-06-10T12:24:23.154274Z","shell.execute_reply":"2021-06-10T12:24:23.160304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # to check if the duplications work \n# new_train[\"cleaned_label\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T12:24:23.162547Z","iopub.execute_input":"2021-06-10T12:24:23.163031Z","iopub.status.idle":"2021-06-10T12:24:23.170588Z","shell.execute_reply.started":"2021-06-10T12:24:23.162997Z","shell.execute_reply":"2021-06-10T12:24:23.169544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train[\"cleaned_label\"].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:35:35.125723Z","iopub.execute_input":"2021-06-11T08:35:35.126043Z","iopub.status.idle":"2021-06-11T08:35:35.137266Z","shell.execute_reply.started":"2021-06-11T08:35:35.126011Z","shell.execute_reply":"2021-06-11T08:35:35.136026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# downsample ","metadata":{}},{"cell_type":"markdown","source":"Uncommend these parts if you want to randomly downsample the largest classes in this data set","metadata":{}},{"cell_type":"code","source":"train[\"cleaned_label\"].value_counts()[:20]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:16.628317Z","iopub.execute_input":"2021-06-11T08:37:16.628655Z","iopub.status.idle":"2021-06-11T08:37:16.63938Z","shell.execute_reply.started":"2021-06-11T08:37:16.628624Z","shell.execute_reply":"2021-06-11T08:37:16.638551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_rows_by_values(df, col, values):\n    return df[df[col].isin(values) == False]\ndownsample_nr = 5\n\ncleaned_counts = train[\"cleaned_label\"].value_counts()[:downsample_nr+1]\nvalues = cleaned_counts.keys().tolist()\ncounts = cleaned_counts.tolist()\nvalues\ncounts","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:21.841343Z","iopub.execute_input":"2021-06-11T08:37:21.841664Z","iopub.status.idle":"2021-06-11T08:37:21.852693Z","shell.execute_reply.started":"2021-06-11T08:37:21.841635Z","shell.execute_reply":"2021-06-11T08:37:21.851822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = counts[downsample_nr]\nsample_size","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:22.377868Z","iopub.execute_input":"2021-06-11T08:37:22.37818Z","iopub.status.idle":"2021-06-11T08:37:22.383051Z","shell.execute_reply.started":"2021-06-11T08:37:22.378151Z","shell.execute_reply":"2021-06-11T08:37:22.382255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_to_downsample = []\nsamples_to_downsample = []\nnewsamples = []\nfor i in range(0,downsample_nr):\n    dlabel = \"label\" + str(i)\n    labels_to_downsample.append(dlabel)\n    dlabelsamp = \"sample\" + str(i)\n    samples_to_downsample.append(dlabelsamp)\n    newsamp = \"sample\" + str(i)\n    newsamples.append(newsamp)\nlabels_to_downsample","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:22.95539Z","iopub.execute_input":"2021-06-11T08:37:22.955718Z","iopub.status.idle":"2021-06-11T08:37:22.964125Z","shell.execute_reply.started":"2021-06-11T08:37:22.955687Z","shell.execute_reply":"2021-06-11T08:37:22.963317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0,downsample_nr):\n    labels_to_downsample[i] = train[(train['cleaned_label']==values[i])]\n","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:23.583108Z","iopub.execute_input":"2021-06-11T08:37:23.583442Z","iopub.status.idle":"2021-06-11T08:37:23.601146Z","shell.execute_reply.started":"2021-06-11T08:37:23.583411Z","shell.execute_reply":"2021-06-11T08:37:23.600221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_appand= []\nfor i in range(0,downsample_nr):\n    to_appand.append(labels_to_downsample[i])\n    print(len(labels_to_downsample[i]))\n# labels_to_downsample[0]['cleaned_label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:24.599937Z","iopub.execute_input":"2021-06-11T08:37:24.600255Z","iopub.status.idle":"2021-06-11T08:37:24.607381Z","shell.execute_reply.started":"2021-06-11T08:37:24.600207Z","shell.execute_reply":"2021-06-11T08:37:24.60634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valuesd = []\nfor i in range(0,downsample_nr):\n    print(values[i])\n    valuesd.append(values[i])\n     ","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:24.921497Z","iopub.execute_input":"2021-06-11T08:37:24.921817Z","iopub.status.idle":"2021-06-11T08:37:24.930453Z","shell.execute_reply.started":"2021-06-11T08:37:24.921786Z","shell.execute_reply":"2021-06-11T08:37:24.929346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df2 = filter_rows_by_values(train,'cleaned_label',valuesd)\nnew_df2[\"cleaned_label\"].value_counts()[:20]\nsamplesize = counts[downsample_nr]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:25.336337Z","iopub.execute_input":"2021-06-11T08:37:25.336647Z","iopub.status.idle":"2021-06-11T08:37:25.35215Z","shell.execute_reply.started":"2021-06-11T08:37:25.336618Z","shell.execute_reply":"2021-06-11T08:37:25.351385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df2[\"cleaned_label\"].value_counts()[:20]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:33.508977Z","iopub.execute_input":"2021-06-11T08:37:33.509316Z","iopub.status.idle":"2021-06-11T08:37:33.518852Z","shell.execute_reply.started":"2021-06-11T08:37:33.509277Z","shell.execute_reply":"2021-06-11T08:37:33.517993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0,downsample_nr):\n    newsamples[i] = labels_to_downsample[i].sample(sample_size)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:33.851087Z","iopub.execute_input":"2021-06-11T08:37:33.851433Z","iopub.status.idle":"2021-06-11T08:37:33.858668Z","shell.execute_reply.started":"2021-06-11T08:37:33.851403Z","shell.execute_reply":"2021-06-11T08:37:33.857794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newsamples[0]['cleaned_label'].value_counts()[:20]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:34.673382Z","iopub.execute_input":"2021-06-11T08:37:34.673697Z","iopub.status.idle":"2021-06-11T08:37:34.681096Z","shell.execute_reply.started":"2021-06-11T08:37:34.673669Z","shell.execute_reply":"2021-06-11T08:37:34.680302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"newsamples.append(new_df2)\nnew_train = pd.concat(newsamples)\nnew_train[\"cleaned_label\"].value_counts()[:20]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:37:35.521602Z","iopub.execute_input":"2021-06-11T08:37:35.521908Z","iopub.status.idle":"2021-06-11T08:37:35.535387Z","shell.execute_reply.started":"2021-06-11T08:37:35.521878Z","shell.execute_reply":"2021-06-11T08:37:35.534522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# load papers","metadata":{}},{"cell_type":"code","source":"papers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:52:28.296884Z","iopub.execute_input":"2021-06-11T07:52:28.29719Z","iopub.status.idle":"2021-06-11T07:52:47.132251Z","shell.execute_reply.started":"2021-06-11T07:52:28.29716Z","shell.execute_reply":"2021-06-11T07:52:47.131268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data to NER format","metadata":{}},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\n# if return true it is a positive datapoint\n# if return false it is a negative. \ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, list(zip(sentence_words, nes))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:52:47.133771Z","iopub.execute_input":"2021-06-11T07:52:47.134081Z","iopub.status.idle":"2021-06-11T07:52:47.148015Z","shell.execute_reply.started":"2021-06-11T07:52:47.134047Z","shell.execute_reply":"2021-06-11T07:52:47.147181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text2(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:52:18.529981Z","iopub.status.idle":"2021-06-11T07:52:18.530704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nner_data = []\n\npbar = tqdm(total=len(new_train))\nfor i, id, dataset_label in new_train[['Id', 'dataset_label']].itertuples():\n    # paper\n    paper = papers[id]\n    \n    # labels\n    labels = dataset_label.split('|')\n    labels = [clean_training_text(label) for label in labels]\n    \n    # sentences\n    sentences = set([clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.') \n                ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    # positive sample\n    for sentence in sentences:\n        is_positive, tags = tag_sentence(sentence, labels)\n        if is_positive:\n            cnt_pos += 1\n            ner_data.append(tags)\n        elif any(word in sentence.lower() for word in ['data', 'study']): \n            prob_not_include = random.randint(1,101)\n            #comment this when you want to downsample the negative instances\n            ner_data.append(tags)\n            cnt_neg += 1\n#           # uncomment this when you want to randomly downsample the negative instances   \n#             if prob_not_include < 50: # change <x percent of what the change is you want to drop the negative instance\n#                 break\n#             else:\n#                 ner_data.append(tags)\n#                 cnt_neg += 1\n    \n    # process bar\n#     pbar.update(1)\n#     pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\n\n# shuffling\nrandom.shuffle(ner_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:52:55.027733Z","iopub.execute_input":"2021-06-11T07:52:55.028051Z","iopub.status.idle":"2021-06-11T07:53:56.638217Z","shell.execute_reply.started":"2021-06-11T07:52:55.028022Z","shell.execute_reply":"2021-06-11T07:53:56.637301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,4))\nx = [\"pos\",\"neg\"]\nplt.bar(x,[cnt_pos, cnt_neg])\nplt.ylable(\"count\")\nplt.title(\"Count positives versus negative\", fontsize = \"xx-large\")","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:00:05.840027Z","iopub.execute_input":"2021-06-11T08:00:05.84038Z","iopub.status.idle":"2021-06-11T08:00:05.977387Z","shell.execute_reply.started":"2021-06-11T08:00:05.840348Z","shell.execute_reply":"2021-06-11T08:00:05.976198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"write data to file.","metadata":{}},{"cell_type":"code","source":"with open('train_ner.json', 'w') as f:\n    for row in ner_data:\n        words, nes = list(zip(*row))\n        row_json = {'tokens' : words, 'tags' : nes}\n        json.dump(row_json, f)\n        f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:00:14.467561Z","iopub.execute_input":"2021-06-11T08:00:14.467877Z","iopub.status.idle":"2021-06-11T08:00:15.236871Z","shell.execute_reply.started":"2021-06-11T08:00:14.467845Z","shell.execute_reply":"2021-06-11T08:00:15.236042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune a BERT model for NER","metadata":{}},{"cell_type":"code","source":"!python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n--model_name_or_path 'bert-base-cased' \\\n--train_file './train_ner.json' \\\n--validation_file './train_ner.json' \\\n--num_train_epochs 1 \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--save_steps 15000 \\\n--output_dir './output' \\\n--report_to 'none' \\\n--seed 123 \\\n--do_train ","metadata":{"execution":{"iopub.status.busy":"2021-06-11T08:00:17.87293Z","iopub.execute_input":"2021-06-11T08:00:17.873273Z","iopub.status.idle":"2021-06-11T08:07:31.411363Z","shell.execute_reply.started":"2021-06-11T08:00:17.873214Z","shell.execute_reply":"2021-06-11T08:07:31.410469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After the tuning finishes, we should find our model in './output'.","metadata":{}}]}