{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Before begins\n\nThis notebook is written in google colab.\n\nTo see some interactive plots, please enter the colab link Below.\n\n<a href=\"https://colab.research.google.com/drive/1iNMV8kik9ue6sy8DPgG73zT7wO_pQKtB?usp=sharing\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open in Colab\"/></a>","metadata":{"id":"RkJL3rKcZ6xq"}},{"cell_type":"markdown","source":"There are many notebooks similar to this for various competitions, so check the github address below\n\n<img src=\"https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png\" width=50 align='left' alt=\"Open in Colab\" /></a>\n&nbsp; <font size=\"5\">[Github: Kaggle-Notebook](https://github.com/JayAhn0104/Kaggle-Notebook)</font>","metadata":{}},{"cell_type":"markdown","source":"# Overview\n\n<br>\n\n## Competition description\n\n<img src=\"https://drive.google.com/uc?export=view&id=1YzoUGRB3lgAZtEt-ImoMBDt-i6RzHLVX\" width=40 align='left' alt=\"Open in Colab\"/></a>\n&nbsp; \n<font size=\"5\">[Dog Breed Identification](https://www.kaggle.com/c/dog-breed-identification/overview)</font>\n\n- Problem type: Multi-class classification for image data (with 3 channels)\n  - Predicting the breed (120 classes) of dogs in the images\n- Evaluation metric: [Multi Class Log Loss](https://www.kaggle.com/c/dog-breed-identification/overview/evaluation)\n\n","metadata":{"id":"xwdEUNQMaMEC"}},{"cell_type":"markdown","source":"# 0. Preliminaries\n\n### > Install Libraries","metadata":{"id":"TKe1Grhwxe0D"}},{"cell_type":"code","source":"%%bash\n\npip install --upgrade --force-reinstall --no-deps kaggle","metadata":{"id":"1_Xp308LrdOq","outputId":"79a320c2-f6ad-4dac-b7f5-a80a515a8925"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\nimport math \nimport shutil\nimport os\nimport torch\nimport torchvision\nfrom torch import nn\nimport numpy as np\n%load_ext tensorboard","metadata":{"id":"HCsJjsISFXGl"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preprocessing","metadata":{"id":"xTODS2fTxUVv"}},{"cell_type":"markdown","source":"## 1-1. Load Data-set","metadata":{"id":"ef1DyAYTxkdo"}},{"cell_type":"markdown","source":"### > Re-organize data files","metadata":{"id":"wRBqduhixrOw"}},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\n    Defined in :numref:`sec_kaggle_cifar10`\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\n    Defined in :numref:`sec_kaggle_cifar10`\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\n    Defined in :numref:`sec_kaggle_cifar10`\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\n    Defined in :numref:`sec_kaggle_cifar10`\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))        ","metadata":{"id":"KFTLNrC9sil_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_dog_data(data_dir, valid_ratio):\n    labels = read_csv_labels(os.path.join(data_dir, 'labels.csv'))\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)\n\ndata_dir = os.path.join('/kaggle/input/dog-breed-identification')\nvalid_ratio = 0.1\nreorg_dog_data(data_dir, valid_ratio)","metadata":{"id":"jp4pSbIEl5EW"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Check sample images","metadata":{"id":"oE-KFmhPca7D"}},{"cell_type":"code","source":"from PIL import Image\nfolder_path = os.path.join(data_dir, 'train')\nimage_list = os.listdir(folder_path)\n\nimages = [np.array(Image.open(os.path.join(folder_path, file))) for file in image_list[:10]]\n\nfrom matplotlib import pyplot as plt\nplt.figure(figsize=(20,10))\ncolumns = 5\nfor i, image in enumerate(images):\n    plt.subplot(len(images) / columns + 1, columns, i + 1)\n    plt.imshow(image)","metadata":{"id":"SAeLb6jFQBI0","outputId":"09eab864-c593-416e-8830-e9d63bd94e7d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1-2. Data Transformation","metadata":{"id":"6wfdvOZQx0dB"}},{"cell_type":"markdown","source":"### > Define a transformation function for Train-set","metadata":{"id":"3VuAeCKNyCEP"}},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    # Randomly crop the image to obtain an image with an area of 0.08 to 1 of\n    # the original area and height-to-width ratio between 3/4 and 4/3. Then,\n    # scale the image to create a new 224 x 224 image\n    torchvision.transforms.RandomResizedCrop(224, scale=(0.08, 1.0),\n                                             ratio=(3.0/4.0, 4.0/3.0)),\n    torchvision.transforms.RandomHorizontalFlip(),\n    # Randomly change the brightness, contrast, and saturation\n    torchvision.transforms.ColorJitter(brightness=0.4,\n                                       contrast=0.4,\n                                       saturation=0.4),\n    torchvision.transforms.ToTensor(),\n    # Standardize each channel of the image\n    torchvision.transforms.Normalize([0.485, 0.456, 0.406],\n                                     [0.229, 0.224, 0.225])])","metadata":{"id":"tmNk8WKCmoY_"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Define a transformation function for Test-set","metadata":{"id":"kldu4eP0yGzG"}},{"cell_type":"code","source":"transform_test = torchvision.transforms.Compose([\n    torchvision.transforms.Resize(256),\n    # Crop a 224 x 224 square area from the center of the image\n    torchvision.transforms.CenterCrop(224),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.485, 0.456, 0.406],\n                                     [0.229, 0.224, 0.225])])","metadata":{"id":"CDcubuKjmqJx"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Transform the Data-set","metadata":{"id":"IkBASq9_ySQF"}},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]","metadata":{"id":"w6BiIpJ6mrjD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of observations \\n'\n    f'train: {len(train_ds)}\\n'\n    f'valid: {len(valid_ds)}\\n'    \n    f'train_valid: {len(train_valid_ds)}\\n'\n    f'test: {len(test_ds)}'\n    )","metadata":{"id":"_26JX53lTYX4","outputId":"0f770aff-c3da-4699-a9da-be65e06d3799"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1-3. Get the data on DataLoader format","metadata":{"id":"XGuBkcePyU7F"}},{"cell_type":"code","source":"batch_size = 128\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"id":"OlrmrtjcxTrA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_to_idx = train_iter.dataset.class_to_idx\nidx_to_class = {class_to_idx[k]:k for k in class_to_idx}","metadata":{"id":"RNkXXFVKXlqQ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Check the transformed data-set","metadata":{"id":"RwnSDFc3dBtU"}},{"cell_type":"code","source":"from matplotlib import pyplot as plt \nimages, labels = next(iter(train_iter))\n\nimages = images[:10]\nlabels = labels[:10]\nlabels_name = [idx_to_class[idx.item()] for idx in labels]\nplt.figure(figsize=(20,10))\ncolumns = 5\nfor i, image in enumerate(images):\n    plt.subplot(len(images) / columns + 1, columns, i + 1)\n    plt.text(5,5, labels_name[i], bbox={'facecolor': 'white', 'pad':10})\n    image = np.transpose(image, (1,2,0))\n    plt.imshow(image)","metadata":{"id":"pHDEpJOx3Eqx","outputId":"cfb7934b-712c-49a8-b146-f80dfbbf92d8"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Model Training (Hyper-parameter Tuning)","metadata":{"id":"n0KeJb_DdGVZ"}},{"cell_type":"markdown","source":"## 2-1. Define some functions needed in Training & Tuning process","metadata":{"id":"8O_2PBA4etOX"}},{"cell_type":"markdown","source":"### > Define a Trainable function that can be compatible with ray.tune","metadata":{"id":"25bil-AsdNrE"}},{"cell_type":"code","source":"from ray import tune\n\ndef Trainable(config, train_loader=None, valid_loader=None, device=None, checkpoint_dir=None):\n  # Model define\n  model = get_net(device)\n  # model.apply(init_weights)\n\n  # Loss function define\n  loss_fn = nn.CrossEntropyLoss(reduction='mean')\n\n  # Optimizer define\n  if config[\"optimizer\"] == \"Adam\":\n    optimizer = torch.optim.Adam((param for param in model.parameters()\n                            if param.requires_grad), \n                         weight_decay=config[\"wd\"], lr=config[\"lr\"])\n  else:\n    optimizer = torch.optim.SGD((param for param in model.parameters()\n                            if param.requires_grad), \n                         weight_decay=config[\"wd\"], lr=config[\"lr\"])\n  \n  # Learning rate scheduler\n  # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, config['lr_period'], config['lr_decay'])\n\n  # Load Checkpoint (if it exist)\n  if checkpoint_dir:\n      model_state, optimizer_state = torch.load(\n          os.path.join(checkpoint_dir, \"checkpoint\"))\n      model.load_state_dict(model_state)\n      optimizer.load_state_dict(optimizer_state)\n\n  ######################################################################\n  # Train & Eval & Save Model\n  ######################################################################\n\n  for epoch in range(config['n_epochs']):\n    # Train model with train_loader\n    tr_loss, tr_acc = 0.0, 0\n    model.train()\n    for i, (X, y) in enumerate(train_loader):\n      X, y = X.to(device), y.to(device)\n      optimizer.zero_grad()\n      output = model(X)\n      l = loss_fn(output, y)\n      l.backward()\n      optimizer.step()\n      with torch.no_grad():\n        tr_acc += torch_accuracy(output, y)\n        tr_loss += l.cpu().numpy()    \n    \n    # Eval Trained model on valid_iter\n    val_loss, val_acc = 0.0, 0\n    model.eval()\n    for i, (X, y) in enumerate(valid_loader):\n      with torch.no_grad():\n        X, y = X.to(device), y.to(device)\n        output = model(X)\n        l = loss_fn(output, y)\n        val_acc += torch_accuracy(output, y)\n        val_loss += l.cpu().numpy()\n    # scheduler.step()\n\n    # Save Checkpoint\n    with tune.checkpoint_dir(epoch) as checkpoint_dir:\n        path = os.path.join(checkpoint_dir, \"checkpoint\")\n        torch.save((model.state_dict(), optimizer.state_dict()), path)\n\n    tune.report(\n        tr_loss=(tr_loss / len(train_loader)), \n        tr_accuracy=(tr_acc / len(train_loader)),\n        val_loss=(val_loss / len(valid_loader)), \n        val_accuracy=(val_acc / len(valid_loader))\n        )\n        ","metadata":{"id":"TAOd5QwvfaNY"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Define some miscellaneous functions","metadata":{"id":"m2T3aJVBeolZ"}},{"cell_type":"code","source":"def torch_accuracy(pred, y):\n  return (pred.argmax(1) == y).type(torch.float).sum().item() / len(y)    \n\ndef try_gpu(i=0): \n    return f'cuda:{i}' if torch.cuda.device_count() >= i + 1 else 'cpu'\n\ndef init_weights(m):\n  if type(m) == nn.Linear or type(m) == nn.Conv2d:\n    nn.init.xavier_uniform_(m.weight)\n    nn.init.constant_(m.bias, 0)    \n\ndef eval_fn(data_iter, net, loss_fn, device=None):\n  loss, acc = 0.0, 0\n  net.eval()\n  for i, (X, y) in enumerate(data_iter):\n    with torch.no_grad():\n      X, y = X.to(device), y.to(device)\n      output = net(X)\n      l = loss_fn(output, y)\n      acc += torch_accuracy(output, y)\n      loss += l.cpu().numpy()\n  return loss/len(data_iter), acc/len(data_iter)    \n\ndef trial_str_creator(trial):\n    return \"{}_{}_123\".format(trial.trainable_name, trial.trial_id)    \n\ndef get_best_model(result, metric=\"val_loss\", mode=\"min\", device=None):\n    best_trial = result.get_best_trial(metric, mode, \"last\")\n    print(\"Best trial config: {}\".format(best_trial.config))\n    print(\"Best trial final validation loss: {}\".format(best_trial.last_result[\"val_loss\"]))\n    print(\"Best trial final validation accuracy: {}\".format(best_trial.last_result[\"val_accuracy\"]))\n\n    best_trained_model = get_net(device)\n\n    best_checkpoint_dir = best_trial.checkpoint.value\n    model_state, optimizer_state = torch.load(os.path.join(best_checkpoint_dir, \"checkpoint\"))\n    best_trained_model.load_state_dict(model_state)\n\n    return best_trained_model, best_trial.config, best_checkpoint_dir","metadata":{"id":"PUoxpCe6hbsB"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Define a model structure\n\n- Model: Pretrained ResNet34 ","metadata":{"id":"NNlNcrVLdVS2"}},{"cell_type":"code","source":"def get_net(device):\n    finetune_net = nn.Sequential()\n    finetune_net.features = torchvision.models.resnet34(pretrained=True)\n    # Define a new output network (there are 120 output categories)\n    finetune_net.output_new = nn.Sequential(nn.Linear(1000, 256),\n                                            nn.ReLU(),\n                                            nn.Linear(256, 120))\n    # Move the model to devices\n    finetune_net = finetune_net.to(device)\n    # Freeze parameters of feature layers\n    for param in finetune_net.features.parameters():\n        param.requires_grad = False\n    return finetune_net","metadata":{"id":"LFNNvnSeZqd4"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2-2. Run Hyper-parameter Tuning (with Training)","metadata":{"id":"z2GR4aeUfXYG"}},{"cell_type":"markdown","source":"### > Define some essential components for Tuning","metadata":{"id":"sS-ctgzofeCR"}},{"cell_type":"code","source":"from ray import tune\nfrom ray.tune import CLIReporter\nfrom ray.tune.schedulers import ASHAScheduler\n\n# args\nargs = {\n    \"local_dir\" : os.path.abspath(\"Tune_Result\"),\n    \"num_samples\" : 6,\n    \"cpus_per_trial\" : 1,\n    \"gpus_per_trial\" : 0.5,\n    \"name\": \"ResNet34_Pretrained\",\n}\n\n# config\n# - define a search space\nconfig = {\n    \"model\": \"ResNet34_Pretrained\",\n    \"wd\": tune.loguniform(1e-3, 1e-1),\n    \"lr\": tune.loguniform(1e-3, 1e-1),\n    'lr_period': 2,\n    'lr_decay': 0.9,\n    \"batch_size\": 128,\n    \"optimizer\": \"Adam\",\n    \"n_epochs\": 20\n    }\n\n# scheduler\nscheduler = ASHAScheduler(\n        metric=\"val_loss\",\n        mode=\"min\",\n        max_t=config[\"n_epochs\"],\n        grace_period=1,\n        reduction_factor=2)\n\n# reporter\nreporter = tune.JupyterNotebookReporter(True,\n    metric_columns=[\"tr_loss\", \"tr_accuracy\", \"val_loss\", \"val_accuracy\", \"training_iteration\"])  \n","metadata":{"id":"jVSi0Vbyhfom"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Run Tune & Get the best trained model","metadata":{"id":"1LiF8DeEf-NN"}},{"cell_type":"code","source":"from functools import partial\n# Run Tune\nresult = tune.run(\n    partial(Trainable, train_loader=train_iter, valid_loader=valid_iter, device=try_gpu()),\n    scheduler=scheduler,\n    progress_reporter=reporter,  \n    config=config,      \n    resources_per_trial={\"cpu\": args[\"cpus_per_trial\"], \"gpu\": args[\"gpus_per_trial\"]},\n    num_samples=args[\"num_samples\"],\n    local_dir=args[\"local_dir\"],\n    name=args[\"name\"],\n    trial_dirname_creator=trial_str_creator,\n    )\n\n# Get the best trained model\nbest_trained_model, best_config, best_checkpoint_dir = get_best_model(result, metric=\"val_loss\", mode=\"min\", device=try_gpu())","metadata":{"id":"HD9EPa_jicZa","outputId":"7d687f89-d152-431f-9e2b-4c13ae702b3d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Visualize the Tuning & Training results with TensorBoard","metadata":{"id":"YKFu361CgCtr"}},{"cell_type":"code","source":"log_dir = os.path.dirname(os.path.dirname(os.path.dirname(best_checkpoint_dir )))\n\n%tensorboard --logdir {log_dir}","metadata":{"id":"kOY-P9Sur06S","outputId":"f6684d96-99d5-441a-9b08-9b9c8c642309"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Make a prediction with the best model","metadata":{"id":"QCXscjSqgZyf"}},{"cell_type":"markdown","source":"## 3-1. Train model on whole Train-set with best hyper-params","metadata":{"id":"XV3PEKDSU3xA"}},{"cell_type":"code","source":"from tqdm import notebook\n\ndevice = try_gpu()\nmodel = get_net(device)\nconfig = best_config\n\n# Loss function define\nloss_fn = nn.CrossEntropyLoss(reduction='mean')\n\n# Optimizer define\nif config[\"optimizer\"] == \"Adam\":\n  optimizer = torch.optim.Adam((param for param in model.parameters()\n                          if param.requires_grad), \n                        weight_decay=config[\"wd\"], lr=config[\"lr\"])\nelse:\n  optimizer = torch.optim.SGD((param for param in model.parameters()\n                          if param.requires_grad), \n                        weight_decay=config[\"wd\"], lr=config[\"lr\"])\n\nfor epoch in notebook.tqdm(range(config['n_epochs'])):\n  tr_loss, tr_acc = 0.0, 0\n  model.train()\n  for i, (X, y) in enumerate(train_valid_iter):\n    X, y = X.to(device), y.to(device)\n    optimizer.zero_grad()\n    output = model(X)\n    l = loss_fn(output, y)\n    l.backward()\n    optimizer.step()\n    with torch.no_grad():\n      tr_acc += torch_accuracy(output, y)\n      tr_loss += l.cpu().numpy()    ","metadata":{"id":"KyCKCUCd0Zix","outputId":"1957e885-5c99-46b4-aba8-5674268890a9"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### > Check the prediction results","metadata":{"id":"DEV_ayzUVBr3"}},{"cell_type":"code","source":"from matplotlib import pyplot as plt \nimages, _ = next(iter(test_iter))\n\ndevice = try_gpu()\npreds = []\noutputs = torch.nn.functional.softmax(model(images.to(device)), dim=0)\npreds.extend(outputs.cpu().detach().numpy())\npred = np.array(preds).argmax(1)\npred_name = [idx_to_class[idx.item()] for idx in pred]\n\nimages = images[:10]\nlabels = labels[:10]\n\nplt.figure(figsize=(20,10))\ncolumns = 5\nfor i, image in enumerate(images):\n    plt.subplot(len(images) / columns + 1, columns, i + 1)\n    # plt.text(5,5, f'label: {labels_name[i]}', bbox={'facecolor': 'white', 'edgecolor': 'red', 'pad':10})\n    plt.text(5, 200, f'pred: {pred_name[i]}', bbox={'facecolor': 'white', 'edgecolor': 'blue', 'pad':10})\n    image = np.array(np.transpose(image, (1,2,0)))\n    plt.imshow(image)","metadata":{"id":"ASspFuEtUw7g","outputId":"1b226556-594f-499a-9f36-81802cb8a8d4"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3-2. Save predictions as submission.csv","metadata":{"id":"t5I6AzIdVHuQ"}},{"cell_type":"code","source":"preds = []\nfor data, label in test_iter:\n    output = torch.nn.functional.softmax(model(data.to(device)), dim=0)\n    preds.extend(output.cpu().detach().numpy())\nids = sorted(os.listdir(\n    os.path.join(data_dir, 'train_valid_test', 'test', 'unknown')))\nwith open('submission.csv', 'w') as f:\n    f.write('id,' + ','.join(train_valid_ds.classes) + '\\n')\n    for i, output in zip(ids, preds):\n        f.write(i.split('.')[0] + ',' + ','.join(\n            [str(num) for num in output]) + '\\n')","metadata":{"id":"6G-HUt3LjG0p"},"execution_count":null,"outputs":[]}]}