{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-01-30T23:15:54.622166Z","iopub.execute_input":"2022-01-30T23:15:54.622690Z","iopub.status.idle":"2022-01-30T23:15:54.632033Z","shell.execute_reply.started":"2022-01-30T23:15:54.622655Z","shell.execute_reply":"2022-01-30T23:15:54.631024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"# !pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:15:54.634435Z","iopub.execute_input":"2022-01-30T23:15:54.634717Z","iopub.status.idle":"2022-01-30T23:15:54.641894Z","shell.execute_reply.started":"2022-01-30T23:15:54.634676Z","shell.execute_reply":"2022-01-30T23:15:54.641027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nfrom tqdm.auto import tqdm\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:15:54.673768Z","iopub.execute_input":"2022-01-30T23:15:54.673995Z","iopub.status.idle":"2022-01-30T23:15:54.681505Z","shell.execute_reply.started":"2022-01-30T23:15:54.673970Z","shell.execute_reply":"2022-01-30T23:15:54.680577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nwandb.init(project=\"pretraining-resnet18\")","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:58:03.042228Z","iopub.execute_input":"2022-01-31T00:58:03.042513Z","iopub.status.idle":"2022-01-31T00:58:09.467527Z","shell.execute_reply.started":"2022-01-31T00:58:03.042484Z","shell.execute_reply":"2022-01-31T00:58:09.466809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"import tarfile\nopened_tar = tarfile.open('/kaggle/input/cifar10/cifar-10/train.xyz')\nopened_tar.extractall('/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:15:54.683519Z","iopub.execute_input":"2022-01-30T23:15:54.684383Z","iopub.status.idle":"2022-01-30T23:16:03.947935Z","shell.execute_reply.started":"2022-01-30T23:15:54.684344Z","shell.execute_reply":"2022-01-30T23:16:03.947215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opened_tar = tarfile.open('/kaggle/input/cifar10/cifar-10/test.xyz')\nopened_tar.extractall('/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:16:03.949202Z","iopub.execute_input":"2022-01-30T23:16:03.949471Z","iopub.status.idle":"2022-01-30T23:17:03.354851Z","shell.execute_reply.started":"2022-01-30T23:16:03.949438Z","shell.execute_reply":"2022-01-30T23:17:03.354068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"# !python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:17:03.356089Z","iopub.execute_input":"2022-01-30T23:17:03.356415Z","iopub.status.idle":"2022-01-30T23:17:03.361725Z","shell.execute_reply.started":"2022-01-30T23:17:03.356378Z","shell.execute_reply":"2022-01-30T23:17:03.359994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:17:03.364427Z","iopub.execute_input":"2022-01-30T23:17:03.364964Z","iopub.status.idle":"2022-01-30T23:17:03.370041Z","shell.execute_reply.started":"2022-01-30T23:17:03.364923Z","shell.execute_reply":"2022-01-30T23:17:03.369114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:17:03.371652Z","iopub.execute_input":"2022-01-30T23:17:03.372332Z","iopub.status.idle":"2022-01-30T23:17:03.379074Z","shell.execute_reply.started":"2022-01-30T23:17:03.372290Z","shell.execute_reply":"2022-01-30T23:17:03.378195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar10/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:17:03.380361Z","iopub.execute_input":"2022-01-30T23:17:03.381402Z","iopub.status.idle":"2022-01-30T23:17:03.684224Z","shell.execute_reply.started":"2022-01-30T23:17:03.381357Z","shell.execute_reply":"2022-01-30T23:17:03.683507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:17:03.685482Z","iopub.execute_input":"2022-01-30T23:17:03.685721Z","iopub.status.idle":"2022-01-30T23:17:03.696027Z","shell.execute_reply.started":"2022-01-30T23:17:03.685687Z","shell.execute_reply":"2022-01-30T23:17:03.695161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:17:03.697307Z","iopub.execute_input":"2022-01-30T23:17:03.697666Z","iopub.status.idle":"2022-01-30T23:17:03.705671Z","shell.execute_reply.started":"2022-01-30T23:17:03.697626Z","shell.execute_reply":"2022-01-30T23:17:03.704971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar10/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:17:03.707078Z","iopub.execute_input":"2022-01-30T23:17:03.707375Z","iopub.status.idle":"2022-01-30T23:17:03.716648Z","shell.execute_reply.started":"2022-01-30T23:17:03.707334Z","shell.execute_reply":"2022-01-30T23:17:03.715836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 256\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:48:00.402765Z","iopub.execute_input":"2022-01-30T23:48:00.403016Z","iopub.status.idle":"2022-01-30T23:49:17.051060Z","shell.execute_reply.started":"2022-01-30T23:48:00.402989Z","shell.execute_reply":"2022-01-30T23:49:17.050318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    torchvision.transforms.RandomHorizontalFlip(p=0.5),\n    torchvision.transforms.RandomResizedCrop(size=(32,32), scale=(0.08, 1.0)),\n    torchvision.transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),\n    \n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])\n\ntransform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:49:17.063570Z","iopub.execute_input":"2022-01-30T23:49:17.063918Z","iopub.status.idle":"2022-01-30T23:49:17.074914Z","shell.execute_reply.started":"2022-01-30T23:49:17.063881Z","shell.execute_reply":"2022-01-30T23:49:17.074174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:49:17.077363Z","iopub.execute_input":"2022-01-30T23:49:17.077716Z","iopub.status.idle":"2022-01-30T23:49:19.457368Z","shell.execute_reply.started":"2022-01-30T23:49:17.077668Z","shell.execute_reply":"2022-01-30T23:49:19.456640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split train between validation and training set\ntrain_size = len(train_ds)\nvalidation_size = len(valid_ds)\ntest_size = len(test_ds)\n\n# After\nprint('\\nTrain dataset size:', train_size)\nprint('Validation dataset size:', validation_size)\nprint('Test dataset size:', validation_size)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:49:19.458498Z","iopub.execute_input":"2022-01-30T23:49:19.458747Z","iopub.status.idle":"2022-01-30T23:49:19.467956Z","shell.execute_reply.started":"2022-01-30T23:49:19.458714Z","shell.execute_reply":"2022-01-30T23:49:19.467248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's see some samples from the dataset","metadata":{}},{"cell_type":"code","source":"# helper function to un-normalize and display an image\ndef imshow(img):\n    img = img / 2 + 0.5  # unnormalize\n    plt.imshow(np.transpose(img, (1, 2, 0)))  # convert from Tensor image\n\nclasses = ('plane', 'car', 'bird', 'cat',\n           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\ndata_iter = iter(train_iter)\nbatch = data_iter.next()\nlabels = batch[1][0:10]\nimages = batch[0][0:10]\nimages = images.numpy() # convert images to numpy for display\n\n# plot the images in the batch, along with the corresponding labels\nfig = plt.figure(figsize=(9, 3), facecolor='white')\n\n# display 10 images\nfor idx in np.arange(10):\n    ax = fig.add_subplot(2, 5, idx+1, xticks=[], yticks=[])\n    imshow(images[idx])\n    ax.set_title(classes[labels[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:49:19.469413Z","iopub.execute_input":"2022-01-30T23:49:19.469691Z","iopub.status.idle":"2022-01-30T23:49:20.136293Z","shell.execute_reply.started":"2022-01-30T23:49:19.469654Z","shell.execute_reply":"2022-01-30T23:49:20.135640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO: \n* Define the data augmentation technique \n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit to Kaggle your submission file and get your accuracy on the test set","metadata":{}},{"cell_type":"markdown","source":"## Run NN on CPU or GPU?","metadata":{}},{"cell_type":"code","source":"# cuda:0, in case of multiple GPUs we will use the first one (0)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint('Detected device: {}'.format(device))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:49:20.137536Z","iopub.execute_input":"2022-01-30T23:49:20.138227Z","iopub.status.idle":"2022-01-30T23:49:20.143646Z","shell.execute_reply.started":"2022-01-30T23:49:20.138191Z","shell.execute_reply":"2022-01-30T23:49:20.142941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_net = torchvision.models.resnet18(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:49:20.145020Z","iopub.execute_input":"2022-01-30T23:49:20.145724Z","iopub.status.idle":"2022-01-30T23:49:20.643055Z","shell.execute_reply.started":"2022-01-30T23:49:20.145686Z","shell.execute_reply":"2022-01-30T23:49:20.642331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ConvNet as fixed feature extractor\nIn Feature Extraction, you will only train the last layer of the pre-train model. The weights of the pre-trained network were not updated during training. Freezing by setting requires_grad = False prevents the weights in a given layer from being updated during training.","metadata":{}},{"cell_type":"code","source":"for param in pretrained_net.parameters():\n    param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:07:57.798458Z","iopub.execute_input":"2022-01-31T00:07:57.799155Z","iopub.status.idle":"2022-01-31T00:07:57.804822Z","shell.execute_reply.started":"2022-01-31T00:07:57.799119Z","shell.execute_reply":"2022-01-31T00:07:57.804002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We then simply swap the head with a `Linear` layer with the correct output shape (= number of classes)","metadata":{}},{"cell_type":"code","source":"num_features = pretrained_net.fc.in_features\npretrained_net.fc = nn.Linear(num_features, len(classes))\n# nn.init.xavier_normal_(pretrained_net.fc.weight)\n# nn.init.constant_(pretrained_net.fc.bias, 0);\n\nprint(len(classes))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:10:52.214088Z","iopub.execute_input":"2022-01-31T00:10:52.214349Z","iopub.status.idle":"2022-01-31T00:10:52.221780Z","shell.execute_reply.started":"2022-01-31T00:10:52.214320Z","shell.execute_reply":"2022-01-31T00:10:52.219624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_net = pretrained_net.to(device)\n\ncriterion = nn.CrossEntropyLoss()\n\n# Observe that all parameters are being optimized\n# optimizer = optim.SGD(pretrained_net.parameters(), lr=0.001, momentum=0.9)\n# Decay LR by a factor of 0.1 every 7 epochs\n# exp_lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n\n# optimizer = torch.optim.Adam(pretrained_net.parameters(), lr = 0.0005) # fine tuned the lr\n\n \n# params_to_update = []\n# for name,param in pretrained_net.named_parameters():\n#     if param.requires_grad == True:\n#         params_to_update.append(param)\n\noptimizer = optim.SGD(params_to_update, lr=0.001, momentum=0.9)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:41:47.115197Z","iopub.execute_input":"2022-01-31T00:41:47.115861Z","iopub.status.idle":"2022-01-31T00:41:47.127836Z","shell.execute_reply.started":"2022-01-31T00:41:47.115820Z","shell.execute_reply":"2022-01-31T00:41:47.125835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 5\nfor epoch in tqdm(range(epochs)):\n    # TRAINING LOOP\n    train_correctly_classified = 0\n    training_total = 0\n    training_losses = 0\n    \n    valid_correctly_classified = 0\n    validation_losses = 0\n    validation_total = 0\n    \n    for X, y in tqdm(train_iter):\n        X = X.to(device)\n        y = y.to(device)\n        # -------------\n        # Forward Pass\n        # -------------\n        # Clear the gradients as we don't want any gradient from previous epoch\n        # to carry forward: don't want to cummulate gradients.\n        # zero the gradients before running\n        # the backward pass.\n        optimizer.zero_grad()\n        # Forward Pass\n        y_pred = pretrained_net(X)\n        # Find the Loss\n        loss = criterion(y_pred, y)\n        \n        training_losses += loss.item()\n        # Backward pass to compute the gradient\n        # of loss w.r.t our learnable params. \n        loss.backward()\n        # Update params\n        optimizer.step()\n        \n        _, predicted = torch.max(y_pred, 1)\n        train_correctly_classified += int(predicted.eq(y).sum().item())\n        training_total += len(y)\n    \n    training_loss = training_losses / len(train_iter)\n    training_accuracy = train_correctly_classified / training_total\n\n    # VALIDATION LOOP\n    with torch.no_grad(): # we don't need gradients in the validation phase\n        for images, labels in valid_iter:\n            images = images.to(device)\n            labels = labels.to(device)\n            # -------------\n            # Forward Pass\n            # -------------\n            output = pretrained_net(images)\n            # Returns the maximum value of all elements in the input tensor (predictions).\n            # dim: the dimension to reduce the tensor to.\n            # _: prediction confidence\n            # predicted: prediction label\n            _, predicted = torch.max(output, dim=1)\n            # Find the Loss\n            loss = criterion(output, labels)\n            # Calculate Loss\n            validation_losses += loss.item()\n            \n            # Comparing the prediction (predicted) with the ground truth (labels)\n            valid_correctly_classified += int(predicted.eq(labels).sum().item())\n            validation_total += len(labels)\n\n    validation_loss = validation_losses / len(valid_iter)\n    validation_accuracy = valid_correctly_classified / validation_total\n    \n    print('Epoch {}'.format(epoch+1),\n        '\\t training_loss: ', training_loss,\n        '\\t training_accuracy: ', training_accuracy,\n        '\\t validation_loss: ', validation_loss,\n        '\\t validation_accuracy: ', validation_accuracy),\n    wandb.log({\n        'training_loss': training_loss,\n        'training_accuracy': training_accuracy,\n        'validation_loss': validation_loss,\n        'validation_accuracy': validation_accuracy})","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:58:24.654646Z","iopub.execute_input":"2022-01-31T00:58:24.655057Z","iopub.status.idle":"2022-01-31T01:02:26.731789Z","shell.execute_reply.started":"2022-01-31T00:58:24.655014Z","shell.execute_reply":"2022-01-31T01:02:26.730950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing","metadata":{}},{"cell_type":"code","source":"correct = 0\ntotal = 0\n# since we're not training, we don't need to calculate the gradients for our outputs\nwith torch.no_grad():\n    for images, labels in test_iter:\n        images = images.to(device)\n        labels = labels.to(device)\n        # calculate outputs by running images through the network\n        output = pretrained_net(images)\n        # the class with the highest energy is what we choose as prediction\n        _, predicted = torch.max(output, dim=1)\n        total += len(labels)\n        correct += int(predicted.eq(labels).sum().item())\n\nprint(f'Accuracy of the network on test images: {100 * correct // total} %')","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:51:05.722111Z","iopub.execute_input":"2022-01-31T00:51:05.722384Z","iopub.status.idle":"2022-01-31T00:53:17.417509Z","shell.execute_reply.started":"2022-01-31T00:51:05.722355Z","shell.execute_reply":"2022-01-31T00:53:17.416192Z"},"trusted":true},"execution_count":null,"outputs":[]}]}