{"cells":[{"metadata":{},"cell_type":"markdown","source":"Here we walk through PyTorch BERT fine-tuning for classification, mostly based on [this Kernel](https://www.kaggle.com/yuval6967/toxic-bert-plain-vanila). \nSome comments on the used heurisitcs (batch accumulation, warmup) are added. \n\n#### Imports"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os, sys, shutil\nimport time\nimport gc\nfrom contextlib import contextmanager\nfrom pathlib import Path\nimport random\nimport numpy as np, pandas as pd\nfrom tqdm import tqdm, tqdm_notebook\n\nfrom sklearn.metrics import f1_score, classification_report\nfrom sklearn.preprocessing import LabelBinarizer\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\n\nfrom matplotlib import pyplot as plt\n%config InlineBackend.figure_formats = ['retina']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 512    # maximal possible sequence length is 512. Generally, the higher, the better, but more GPU memory consumed\nBATCH_SIZE = 16              # refer to the table here https://github.com/google-research/bert to adjust batch size to seq length\n\nSEED = 1234\nEPOCHS = 2                                   \nPATH_TO_DATA = Path(\"../input/clickbait-news-detection/\")\nWORK_DIR = Path(\"../working/\")\n\nLRATE = 2e-5             # hard to tune with BERT, but this shall be fine (improvements: LR schedules, fast.ai wrapper for LR adjustment)\nACCUM_STEPS = 2          # wait for several backward steps, then one optimization step, this allows to use larger batch size\n                         # well explained here https://medium.com/huggingface/training-larger-batches-practical-tips-on-1-gpu-multi-gpu-distributed-setups-ec88c3e51255\nWARMUP = 5e-2            # warmup helps to tackle instability in the initial phase of training with large learning rates. \n                         # During warmup, learning rate is gradually increased from 0 to LRATE.\n                         # WARMUP is a proportion of total weight updates for which warmup is done. By default, it's linear warmup\nUSE_APEX = True          # using APEX shall speedup training (here we use mixed precision training), https://github.com/NVIDIA/apex","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# nice way to report running times\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.0f} s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make results fully reproducible\ndef seed_everything(seed=123):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if USE_APEX:\n    with timer('install Nvidia apex'):\n        # Installing Nvidia Apex\n        os.system('git clone https://github.com/NVIDIA/apex; cd apex; pip install -v --no-cache-dir' + \n                  ' --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./')\n        os.system('rm -rf apex/.git') # too many files, Kaggle fails\n        from apex import amp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Download BERT model and convert it to PyTorch format"},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget -q https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip\n!unzip uncased_L-12_H-768_A-12.zip\nBERT_MODEL_PATH = Path('uncased_L-12_H-768_A-12/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture\n# Add the Bert Pytorch repo to the PATH using files from: https://github.com/huggingface/pytorch-pretrained-BERT\npackage_dir_a = \"../input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT\"\nsys.path.insert(0, package_dir_a)\n\nfrom pytorch_pretrained_bert import convert_tf_checkpoint_to_pytorch\nfrom pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification, BertAdam, BertConfig\n\n# Translate model from tensorflow to pytorch\nconvert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(\n    str(BERT_MODEL_PATH / 'bert_model.ckpt'),\n    str(BERT_MODEL_PATH / 'bert_config.json'),\n    str(WORK_DIR / 'pytorch_model.bin')\n)\n\nshutil.copyfile(BERT_MODEL_PATH / 'bert_config.json', WORK_DIR / 'bert_config.json')\nbert_config = BertConfig(str(BERT_MODEL_PATH / 'bert_config.json'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Read and convert data \nKudos to [this Kernel](https://www.kaggle.com/httpwwwfszyc/bert-in-keras-taming). "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting the lines to BERT format\ndef convert_lines(example, max_seq_length, tokenizer):\n    max_seq_length -= 2\n    all_tokens, lengths = [], []\n    longer = 0\n    for text in tqdm_notebook(example):\n        tokens_a = tokenizer.tokenize(text)\n        lengths.append(len(tokens_a))\n        if len(tokens_a) > max_seq_length:\n            tokens_a = tokens_a[:max_seq_length]\n            longer += 1\n        one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"] + tokens_a + [\"[SEP]\"]) + [0] * (max_seq_length - len(tokens_a))\n        all_tokens.append(one_token)\n    print(f\"There are {longer} lines longer than {max_seq_length}\")\n    return np.array(all_tokens), np.array(lengths)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following simple bucketing is taken from [this Kernel](https://www.kaggle.com/authman/simple-lstm-pytorch-with-batch-loading) - Simple LSTM (PyTorch version) for Jigsaw competition.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing import sequence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TextDataset(torch.utils.data.Dataset):\n    def __init__(self, text, lens, y=None):\n        self.text = text\n        self.lens = lens\n        self.y = y\n\n    def __len__(self):\n        return len(self.lens)\n\n    def __getitem__(self, idx):\n        if self.y is None:\n            return self.text[idx], self.lens[idx]\n        return self.text[idx], self.lens[idx], self.y[idx]\n    \n    \nclass Collator(object):\n    def __init__(self,test=False,percentile=100):\n        self.test = test\n        self.percentile = percentile\n        \n    def __call__(self, batch):\n        global MAX_SEQUENCE_LENGTH\n        \n        if self.test:\n            texts, lens = zip(*batch)\n        else:\n            texts, lens, target = zip(*batch)\n\n        lens = np.array(lens)\n        max_len = min(int(np.percentile(lens, self.percentile)), MAX_SEQUENCE_LENGTH)\n        texts = torch.tensor(sequence.pad_sequences(texts, maxlen=max_len), dtype=torch.long).cuda()\n        \n        if self.test:\n            return texts\n        \n        return texts, torch.tensor(target, dtype=torch.float32).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer('Read data and convert to BERT format'):\n    tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,\n                                              do_lower_case=True)\n    train_df = pd.read_csv(PATH_TO_DATA / \"train.csv\")\n\n    valid_df = pd.read_csv(PATH_TO_DATA / \"valid.csv\")\n    \n    train_val_df = pd.concat([train_df, valid_df])\n    \n    test_df = pd.read_csv(PATH_TO_DATA / \"test.csv\")\n    \n    print('loaded {} train + validation and {} test records'.format(\n        len(train_val_df), len(test_df)))\n\n    # Make sure all text values are strings\n    train_val_df['text'] = train_val_df['title'].astype(str).fillna(\"DUMMY_VALUE\") + '_' + train_val_df['text'].astype(str).fillna(\"DUMMY_VALUE\") \n    test_df['text'] = test_df['title'].astype(str).fillna(\"DUMMY_VALUE\") + '_' + test_df['text'].astype(str).fillna(\"DUMMY_VALUE\") \n    \n    label_binarizer = LabelBinarizer()\n    y_train = label_binarizer.fit_transform(train_val_df['label'])\n    \n    X_train, train_lengths = convert_lines(train_val_df[\"text\"], MAX_SEQUENCE_LENGTH, tokenizer)\n    X_test, test_lengths = convert_lines(test_df[\"text\"], MAX_SEQUENCE_LENGTH, tokenizer)\n    \n    train_collate = Collator(percentile=96)\n    train_dataset = TextDataset(X_train, train_lengths, y_train)\n    train_loader  = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, \n                                                shuffle=True, collate_fn=train_collate)\n    \n    test_collate = Collator(test=True)\n    test_dataset = TextDataset(X_test, test_lengths)\n    test_loader  = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE,\n                                               shuffle=False , collate_fn=test_collate)\n    \n#     del train_val_df, train_df, valid_df, test_df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Setting up BERT"},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer('Setting up BERT'):\n    output_model_file = \"bert_pytorch.bin\"\n    seed_everything(SEED)\n    model = BertForSequenceClassification.from_pretrained(WORK_DIR,\n                                                          cache_dir=None,\n                                                          num_labels=y_train.shape[1])\n    model.zero_grad()\n    model = model.to(device)\n    param_optimizer = list(model.named_parameters())\n    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n    optimizer_grouped_parameters = [\n        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], \n         'weight_decay': 0.01},\n        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], \n         'weight_decay': 0.0}\n        ]\n\n    num_train_optimization_steps = int(EPOCHS * len(train_dataset) / BATCH_SIZE / ACCUM_STEPS)\n\n    optimizer = BertAdam(optimizer_grouped_parameters,\n                         lr=LRATE, warmup=WARMUP,\n                         t_total=num_train_optimization_steps)\n    if USE_APEX:\n        model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\",verbosity=0)\n    model = model.train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer('Training'):\n    loss_history = []\n    tq = tqdm_notebook(range(EPOCHS))\n    for epoch in tq:\n        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, \n                                                shuffle=True, collate_fn=train_collate)\n        avg_loss = 0.\n        lossf = None\n        tk0 = tqdm_notebook(enumerate(train_loader), total=len(train_loader), leave=False)\n        optimizer.zero_grad()   \n        for i,(x_batch, y_batch) in tk0:\n            y_pred = model(x_batch.to(device), \n                           attention_mask=(x_batch>0).to(device), labels=None)\n            loss = nn.BCEWithLogitsLoss()(y_pred, y_batch.to(device))\n            if USE_APEX:\n                with amp.scale_loss(loss, optimizer) as scaled_loss:\n                    scaled_loss.backward()\n            else:\n                loss.backward()\n            if (i+1) % ACCUM_STEPS == 0:                     # Wait for several backward steps\n                optimizer.step()                             # Now we can do an optimizer step\n                optimizer.zero_grad()\n            if lossf:\n                lossf = 0.98 * lossf + 0.02 * loss.item()\n            else:\n                lossf = loss.item()\n            tk0.set_postfix(loss = lossf)\n            avg_loss += loss.item() / len(train_loader)\n            \n            loss_history.append(loss.item())\n            \n        tq.set_postfix(avg_loss=avg_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(range(len(loss_history)), loss_history);\nplt.ylabel('Loss'); plt.xlabel('Batch number');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Test predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer('Test predictions'):\n    # The following 3 lines are not needed but show how to download the model for prediction\n#     model = BertForSequenceClassification(bert_config, num_labels=y_train.shape[1])\n#     model.load_state_dict(torch.load(output_model_file))\n#     model.to(device)\n    for param in model.parameters():\n        param.requires_grad = False\n    model.eval();\n    \n    test_pred_probs = np.zeros((len(X_test), y_train.shape[1]))\n        \n    for i, x_batch in enumerate(tqdm_notebook(test_loader)):\n        pred = model(x_batch.to(device), attention_mask=(x_batch>0).to(device), labels=None)\n        test_pred_probs[i*BATCH_SIZE:(i+1)*BATCH_SIZE] = pred.detach().cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Creating a submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = label_binarizer.inverse_transform(test_pred_probs)\n\nsub_df = pd.read_csv(PATH_TO_DATA / 'sample_submission.csv', index_col='id')\nsub_df['label'] = test_preds \nsub_df.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"00eb48031aa74e3589076cea7e4f79a9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"0c1e14f9485941f788330f69df45cd7c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0fb6cee21e0a4c4c9f36b2c08590e9bf":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"106a04d1cf6f4d059cbff70f9a264f72":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"19b6edf9d5fc499c856a35d983c7503e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"21ddb55eac8149f6b5d4d7d54a99bf9a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"26b12826f8354f7a82d6f2c7c9bd75fc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"2ed936c1d4614444bef244ccd424f77c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_60dff47ebbd94032a9e3d52afcb5ee34","placeholder":"​","style":"IPY_MODEL_26b12826f8354f7a82d6f2c7c9bd75fc","value":"100% 1563/1563 [06:03<00:00,  5.02it/s]"}},"3c3dce72dc854c27b9d371a920eebacc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_df4af09d60b84cc5a3e3a4c2379bf894","IPY_MODEL_2ed936c1d4614444bef244ccd424f77c"],"layout":"IPY_MODEL_21ddb55eac8149f6b5d4d7d54a99bf9a"}},"4ba817d8cbdf4c4f9d22742863109e66":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"60dff47ebbd94032a9e3d52afcb5ee34":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"626a6ea61f454888af9e8d38f297be49":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"661e31cf4d1d43dcb5f8755f7bd86859":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_106a04d1cf6f4d059cbff70f9a264f72","placeholder":"​","style":"IPY_MODEL_626a6ea61f454888af9e8d38f297be49","value":"100% 150000/150000 [02:54<00:00, 858.98it/s]"}},"6e09164390c34f338e9ed77070aff11b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_0fb6cee21e0a4c4c9f36b2c08590e9bf","max":3125,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c4b56f4074b74cfbbc3953f61b10cb2c","value":3125}},"73dafc80e33648bb943c284220a2ebd4":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_877dfeb7364143e79c134146433584e9","IPY_MODEL_661e31cf4d1d43dcb5f8755f7bd86859"],"layout":"IPY_MODEL_0c1e14f9485941f788330f69df45cd7c"}},"7aa3334b14cc411ba1efaecfb2faa8db":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7b81182f12b545b098915ce774c37023":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7eb1d4387200493694646ae737ae678c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_19b6edf9d5fc499c856a35d983c7503e","placeholder":"​","style":"IPY_MODEL_00eb48031aa74e3589076cea7e4f79a9","value":"100% 1/1 [36:07<00:00, 2167.06s/it, avg_accuracy=0.961, avg_loss=0.416]"}},"85313e8e32614af78f5a522e490de91a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"877dfeb7364143e79c134146433584e9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_7b81182f12b545b098915ce774c37023","max":150000,"min":0,"orientation":"horizontal","style":"IPY_MODEL_bb168e18f46b434ea941437878be2459","value":150000}},"a170f7ad64f84709a1f8049ff3963cd1":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f6c685a13cd54bfe8415330a66eb1a88","IPY_MODEL_7eb1d4387200493694646ae737ae678c"],"layout":"IPY_MODEL_ce3b44c11de64dbb8cb6e16a8bb3fbae"}},"b505fc43b335496eb83bbfb44035f842":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7aa3334b14cc411ba1efaecfb2faa8db","placeholder":"​","style":"IPY_MODEL_85313e8e32614af78f5a522e490de91a","value":"100% 3125/3125 [36:07<00:00,  1.45it/s, loss=0.399]"}},"bb168e18f46b434ea941437878be2459":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c22e11db1b20401d8e870b8c09f6e4fe":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c4b56f4074b74cfbbc3953f61b10cb2c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c6144ca29cae4ae5b5188966638b458c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_6e09164390c34f338e9ed77070aff11b","IPY_MODEL_b505fc43b335496eb83bbfb44035f842"],"layout":"IPY_MODEL_4ba817d8cbdf4c4f9d22742863109e66"}},"c976c032987d466f85df9766c23a7486":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"ce3b44c11de64dbb8cb6e16a8bb3fbae":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d6f75126015f4151859512a99978a11b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"df4af09d60b84cc5a3e3a4c2379bf894":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_c22e11db1b20401d8e870b8c09f6e4fe","max":1563,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c976c032987d466f85df9766c23a7486","value":1563}},"eec4cd4cac31475a8d2e3341887ec96f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f6c685a13cd54bfe8415330a66eb1a88":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_eec4cd4cac31475a8d2e3341887ec96f","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_d6f75126015f4151859512a99978a11b","value":1}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}