{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nfrom torch.utils.data.dataset import Dataset\nimport torchvision\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch.nn as nn\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:36.379217Z","iopub.execute_input":"2021-07-29T18:24:36.379552Z","iopub.status.idle":"2021-07-29T18:24:36.386279Z","shell.execute_reply.started":"2021-07-29T18:24:36.379524Z","shell.execute_reply":"2021-07-29T18:24:36.385381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#* Importing dataset\ndataset_train = pd.read_csv('../input/digit-recognizer/train.csv')\ndataset_test = pd.read_csv('../input/digit-recognizer/test.csv')\nprint('Training data length:', len(dataset_train))","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:36.387652Z","iopub.execute_input":"2021-07-29T18:24:36.388152Z","iopub.status.idle":"2021-07-29T18:24:40.139972Z","shell.execute_reply.started":"2021-07-29T18:24:36.388114Z","shell.execute_reply":"2021-07-29T18:24:40.138523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#* Having a look at the dataset\ntargets_np = dataset_train.label.values\nfeatures_np = dataset_train.loc[:,dataset_train.columns!='label'].values # values range to 255, so to normalize\nX_train, X_test, y_train, y_test = train_test_split(features_np,targets_np,test_size=0.2, random_state=42)\n#features_train = torch.from_numpy(X_train)\n#target_train = torch.from_numpy(y_train)\n#features_test = torch.from_numpy(X_test)\n#target_test = torch.from_numpy(y_test)\nprint('Training Features Shape',X_train.shape)\nprint('Training Labels Shape',y_train.shape)\nprint('Testing Features Shape',X_test.shape)\nprint('Testing Labels Shape',y_test.shape)\n#Viz of an image\nplt.imshow(features_np[0].reshape(28,28),cmap='gray') # images are 28x28\nplt.title('Label: '+str(targets_np[0]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:40.141816Z","iopub.execute_input":"2021-07-29T18:24:40.142175Z","iopub.status.idle":"2021-07-29T18:24:40.734940Z","shell.execute_reply.started":"2021-07-29T18:24:40.142137Z","shell.execute_reply":"2021-07-29T18:24:40.733983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=100\nplt.imshow(X_test[i].reshape(28,28),cmap='gray') # images are 28x28\nplt.title('Label: '+str(y_test[i]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:40.736785Z","iopub.execute_input":"2021-07-29T18:24:40.737144Z","iopub.status.idle":"2021-07-29T18:24:40.856331Z","shell.execute_reply.started":"2021-07-29T18:24:40.737108Z","shell.execute_reply":"2021-07-29T18:24:40.855382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as tt\n\ndef get_default_device():\n    \"\"\"Returns the default Device (cuda if gpu, else cpu\"\"\"\n    if torch.cuda.is_available(): return torch.device('cuda')\n    else: torch.device('cpu')\n\ndef to_device(data,device):\n    \"\"\"Shifts Data to the input device\"\"\"\n    if isinstance(data,(list,tuple)): return [to_device(x,device) for x in data]\n    return data.to(device, non_blocking=True)\n\nclass DeviceDataLoader():\n\n    def __init__(self, dl, device) -> None:\n        self.dl = dl\n        self.device = device\n\n    def __iter__(self):\n        for b in self.dl:\n            yield to_device(b,self.device)\n    \n    def __len__(self): return len(self.dl)\n\n\n#* Creating a custom dataset overriding torchvision dataset\nclass MyDataset(Dataset):\n    def __init__(self, df_features, df_labels, transform=None) -> None:\n        self.features = df_features\n        self.labels = df_labels\n        self.transform = transform\n    \n    def __getitem__(self, index):\n        data = self.features[index]\n        image = data.reshape(28,28).astype(np.uint8)\n        label = self.labels[index]\n        if self.transform:\n            image = self.transform(image)\n        return (image,label)\n\n    def __len__(self): return len(self.features)\n\nBATCH_SIZE = 16\n\n#* Augmentation - Random affination (rotation with translation), normalization\ntrain_tmfs = tt.Compose([\n    tt.ToPILImage(),\n    #tt.RandomRotation(30), \n    tt.RandomAffine(degrees=30, translate = (0.1,0.1), scale = (0.9,1.1)),\n    # Random Affine suits better for this purpose @https://pytorch.org/vision/stable/auto_examples/plot_transforms.html#randomaffine\n    tt.ToTensor(),\n    tt.Normalize(mean=[features_np.mean()/255], std=[features_np.std()/255],inplace=True)\n])\ntest_tmfs = tt.Compose([\n    tt.ToPILImage(),\n    #tt.RandomRotation(30),\n    tt.RandomAffine(degrees=30, translate = (0.1,0.1), scale = (0.9,1.1)),\n    # Random Affine suits better for this purpose @https://pytorch.org/vision/stable/auto_examples/plot_transforms.html#randomaffine\n    tt.ToTensor(),\n    tt.Normalize(mean=[features_np.mean()/255], std=[features_np.std()/255],inplace=True)\n])\n\n#* Setting default Device, GPU\ndevice = get_default_device()\nprint(device)\n\n#* Creating Dataset\ntrain_set = MyDataset(X_train, y_train, transform=train_tmfs)\ntest_set = MyDataset(X_test, y_test, transform=test_tmfs)\n\n#* Loading data and then shifting data loader to the selected device\ntrain_dl = DeviceDataLoader(DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True),device)\ntest_dl = DeviceDataLoader(DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True),device)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:40.857649Z","iopub.execute_input":"2021-07-29T18:24:40.858024Z","iopub.status.idle":"2021-07-29T18:24:41.352733Z","shell.execute_reply.started":"2021-07-29T18:24:40.857990Z","shell.execute_reply":"2021-07-29T18:24:41.351880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#* Sanity Check\nimgs, lbls = next(iter(test_dl))\nprint(imgs.shape)\nplt.imshow(torch.Tensor.cpu(imgs[0].data.reshape(28,28)),cmap='gray')\nprint('Label:',lbls[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:41.354059Z","iopub.execute_input":"2021-07-29T18:24:41.354438Z","iopub.status.idle":"2021-07-29T18:24:45.776041Z","shell.execute_reply.started":"2021-07-29T18:24:41.354400Z","shell.execute_reply":"2021-07-29T18:24:45.775061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F\n#* Following Residual Network https://raw.githubusercontent.com/lambdal/cifar10-fast/master/net.svg\nNUM_CLASSES = 10 \n\ndef convolution_block(in_channels, out_channels, pool=False):\n    \"\"\"Returns a sequence of Convolution, Batch Normalization, Rectified Linear Activation (Relu) and Max Pooling if selected\"\"\"\n    layers = [\n        nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, padding=1),\n        nn.BatchNorm2d(out_channels),\n        nn.ReLU(inplace=True)\n    ]\n    if pool: layers.append(nn.MaxPool2d(2,2)) # 2x2 Max Pooling\n    return nn.Sequential(*layers)\n\nclass Model(nn.Module):\n\n    def __init__(self):\n        super().__init__()\n\n        self.conv1 = convolution_block(1,32)\n        \n        self.conv2 = convolution_block(32,64,pool=True)\n        self.res1 = nn.Sequential(convolution_block(64,64), convolution_block(64,64))\n        \n        self.conv3 = convolution_block(64,128, pool=True)\n\n        self.conv4 = convolution_block(128,256, pool=True)\n        self.res2 = nn.Sequential(convolution_block(256,256), convolution_block(256,256))\n\n        self.classifier = nn.Sequential(\n            nn.MaxPool2d(2),\n            nn.Flatten(),\n            nn.Dropout(0.2), #* added later\n            nn.Linear(256, NUM_CLASSES)\n        )\n    \n    def forward(self, xb):\n        out = self.conv1(xb) # Prep\n        out = self.conv2(out) # Layer 1 Convolution Block\n        out = self.res1(out) + out # Layer 1 Residual Block\n        out = self.conv3(out) # Layer 2\n        out = self.conv4(out) # Layer 3 Convolution Block\n        out = self.res2(out) + out # Layer 3 Residual Block\n        out = self.classifier(out)\n        return F.log_softmax(out, dim=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:45.778330Z","iopub.execute_input":"2021-07-29T18:24:45.778798Z","iopub.status.idle":"2021-07-29T18:24:45.799542Z","shell.execute_reply.started":"2021-07-29T18:24:45.778762Z","shell.execute_reply":"2021-07-29T18:24:45.798739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model()\nto_device(model,device)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:45.805553Z","iopub.execute_input":"2021-07-29T18:24:45.808129Z","iopub.status.idle":"2021-07-29T18:24:45.862425Z","shell.execute_reply.started":"2021-07-29T18:24:45.808091Z","shell.execute_reply":"2021-07-29T18:24:45.861654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sanity Check\nfor images, labels in train_dl:\n    print(images.shape)\n    out = model(images)\n    print(out.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:45.866702Z","iopub.execute_input":"2021-07-29T18:24:45.869022Z","iopub.status.idle":"2021-07-29T18:24:46.857621Z","shell.execute_reply.started":"2021-07-29T18:24:45.868984Z","shell.execute_reply":"2021-07-29T18:24:46.856730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nCant find fastai modules\nfrom fastai.basic_data import DataBunch\nfrom fastai.train import Learner\nfrom fastai.metrics import accuracy\n'''","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:46.858844Z","iopub.execute_input":"2021-07-29T18:24:46.859345Z","iopub.status.idle":"2021-07-29T18:24:46.865205Z","shell.execute_reply.started":"2021-07-29T18:24:46.859306Z","shell.execute_reply":"2021-07-29T18:24:46.864363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n@torch.no_grad()\ndef loss_batch(model, loss_func, xb,yb, opt=None, metric=None):\n    preds = model(xb)\n    loss = loss_func(preds,yb)\n\n    if opt is not None:\n        loss.backward()\n        opt.step()\n        opt.zero_grad()\n\n    metric_result = None\n    if metric is not None:\n        metric_result = metric(preds, yb)\n\n    return loss.item(), len(xb), metric_result\n\n\ndef accuracy(outputs, labels):\n    _, preds = torch.max(outputs, dim=1)\n    return torch.tensor(torch.sum(preds == labels).item() / len(preds))\n\ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n    \ndef fit_one_cycle(epochs, max_lr, model, train_dl, test_dl, \n                    weight_decay = 0, grad_clip = None, opt_func = torch.optim.SGD):\n\n    def evaluate(model, test_dl):\n        model.eval()\n        outputs = [validation_step(images.to(device),labels.to(device)) for images, labels in test_dl]\n        return validation_epoch_end(outputs)\n\n    def training_step(images, labels):\n        out = model(images)\n        loss = F.cross_entropy(out,labels)\n        return loss\n\n    def validation_step(images, labels):\n        out = model(images)\n        loss = F.cross_entropy(out,labels)\n        acc = accuracy(out, labels)\n        return {'val_loss': loss.detach(), 'val_acc':acc}\n    \n    def validation_epoch_end(outputs):\n        batch_losses = [x['val_loss'] for x in outputs]\n        epoch_losses = torch.stack(batch_losses).mean()\n        batch_accs = [x['val_acc'] for x in outputs]\n        epoch_acc = torch.stack(batch_accs).mean()\n        return {'val_loss': epoch_losses.item(), 'val_acc':epoch_acc.item()}\n    \n    def epoch_end(epoch, result, start_time):\n        print(\"Epoch [{}], last_lr:{:.5f}, train_loss:{:.4f}, val_loss:{:.4f}, val_acc: {:.4f}, epoch_time:{:.4f}\".format(\n            epoch, result['lrs'][-1], result['train_loss'], result['val_loss'], result['val_acc'], time.time()-start_time\n        ))\n\n    torch.cuda.empty_cache()\n    results = []\n    optimizer = opt_func(model.parameters(), max_lr, weight_decay=weight_decay)\n    scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr, epochs=epochs, \n                                            steps_per_epoch=len(train_dl))\n\n    for epoch in range(epochs):\n        epoch_start_time = time.time()\n        model.train()\n        train_losses = []\n        lrs = [] # Learning rates at each point\n        #* Training Part\n        for batch in train_dl:\n            images, labels = batch\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            loss = training_step(images, labels) # Generate Predictions and Calculate the loss\n            train_losses.append(loss)\n            loss.backward()\n\n            if grad_clip:\n                nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n            \n            optimizer.step() # Compute Gradients\n            optimizer.zero_grad() # Reset to zero\n\n            lrs.append(get_lr(optimizer))\n            scheduler.step()\n        \n        #* Validation part\n        result = evaluate(model, train_dl)\n        result['train_loss'] = torch.stack(train_losses).mean().item()\n        result['lrs'] = lrs\n        epoch_end(epoch, result, epoch_start_time)\n        results.append(result)\n    \n    return results","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:46.866794Z","iopub.execute_input":"2021-07-29T18:24:46.867387Z","iopub.status.idle":"2021-07-29T18:24:46.887629Z","shell.execute_reply.started":"2021-07-29T18:24:46.867351Z","shell.execute_reply":"2021-07-29T18:24:46.886666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 10\nMAX_LEARNING_RATE = 0.001\nGRAD_CLIP = 0.1\nWEIGHT_DECAY = 1e-4\nOPT_FUNC = torch.optim.Adam","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:46.888987Z","iopub.execute_input":"2021-07-29T18:24:46.889361Z","iopub.status.idle":"2021-07-29T18:24:46.900866Z","shell.execute_reply.started":"2021-07-29T18:24:46.889325Z","shell.execute_reply":"2021-07-29T18:24:46.900023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\nresults += fit_one_cycle(EPOCHS, MAX_LEARNING_RATE, model, train_dl, test_dl, WEIGHT_DECAY, GRAD_CLIP, OPT_FUNC)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:24:46.902099Z","iopub.execute_input":"2021-07-29T18:24:46.902456Z","iopub.status.idle":"2021-07-29T18:34:46.150781Z","shell.execute_reply.started":"2021-07-29T18:24:46.902422Z","shell.execute_reply":"2021-07-29T18:34:46.149789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#* Plotting training history\ntime_taken = '8:19'\nfig, axs = plt.subplots(nrows=1,ncols=3,figsize=(30,5))\naccuracies = [x['val_acc'] for x in results]\naxs[0].plot(accuracies, '-x')\naxs[0].title.set_text('Accuracies x Number of Epochs')\naxs[0].set_ylabel('Accuracies')\naxs[0].set_xlabel('Epoch')\naxs[1].plot([x['val_loss'] for x in results], '-x')\naxs[1].title.set_text('Loss x Number of Epochs')\naxs[1].set_ylabel('Loss')\naxs[1].set_xlabel('Epoch')\naxs[1].plot([x['train_loss'] for x in results], '-rx')\naxs[1].title.set_text('Training Loss x Number of Epochs')\nlrs = np.concatenate([x.get('lrs',[]) for x in results])\naxs[2].plot(lrs)\naxs[2].title.set_text('Learning Rate x Batch Number')\naxs[2].set_ylabel('Learning Rate')\naxs[2].set_ylabel('Batch Number')\nplt.suptitle('Training History')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:34:46.179864Z","iopub.execute_input":"2021-07-29T18:34:46.180189Z","iopub.status.idle":"2021-07-29T18:34:46.548238Z","shell.execute_reply.started":"2021-07-29T18:34:46.180157Z","shell.execute_reply":"2021-07-29T18:34:46.547412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalizing testing images\ndef normalize_test(X_test):\n    X_test = X_test.reshape([-1,28,28]).astype(float)/255\n    X_test = (X_test-features_np.mean())/features_np.std()\n    return X_test\nX_test = normalize_test(X_test)\n# expanding dimension of test\nX_test = np.expand_dims(X_test,axis=1)\nX_test = torch.from_numpy(X_test).float().to(device)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:34:46.549400Z","iopub.execute_input":"2021-07-29T18:34:46.549810Z","iopub.status.idle":"2021-07-29T18:34:46.870935Z","shell.execute_reply.started":"2021-07-29T18:34:46.549772Z","shell.execute_reply":"2021-07-29T18:34:46.870038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testing some images, sanity checkk\nfor img, lbl in test_dl:\n    ttt=img,lbl\n    for i in range(BATCH_SIZE):\n        plt.imshow(torch.Tensor.cpu(ttt[0][i].data.reshape(28,28)),cmap='gray')\n        plt.show()\n        print('Prediction',torch.argmax(model(ttt[0][i].unsqueeze(0)),1).item())\n    break\n# Seems to working :)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:34:46.872167Z","iopub.execute_input":"2021-07-29T18:34:46.872546Z","iopub.status.idle":"2021-07-29T18:34:48.744140Z","shell.execute_reply.started":"2021-07-29T18:34:46.872510Z","shell.execute_reply":"2021-07-29T18:34:48.743252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#* Prepping submission file\nsubmission_np = dataset_test.values\nclass SubmissionDataset(Dataset):\n    def __init__(self, df, transform=None) -> None:\n        self.features = df\n        self.transform = transform\n    \n    def __getitem__(self, index):\n        data = self.features[index]\n        image = data.reshape(28,28).astype(np.uint8)\n        if self.transform:\n            image = self.transform(image)\n        return image\n\n    def __len__(self): return len(self.features)\n\nsubmission_tmfs = tt.Compose([\n    tt.ToTensor(),\n    tt.Normalize(mean=[features_np.mean()/255], std=[features_np.std()/255],inplace=True)\n])\n\nsubmission_set = SubmissionDataset(submission_np,transform=submission_tmfs)\nsubmission_dl = DeviceDataLoader(DataLoader(submission_set, batch_size=1, shuffle=False),device)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:34:48.745519Z","iopub.execute_input":"2021-07-29T18:34:48.746136Z","iopub.status.idle":"2021-07-29T18:34:49.044563Z","shell.execute_reply.started":"2021-07-29T18:34:48.746092Z","shell.execute_reply":"2021-07-29T18:34:49.043559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#* Checking image by image in submission set\nctr=0\nfor img in submission_dl:\n    for i in range(1):\n        print(img[i].shape)\n        plt.imshow(torch.Tensor.cpu(img[i].data.reshape(28,28)),cmap='gray')\n        plt.show()\n        print('Prediction',int(torch.argmax(model(img[i].unsqueeze(0)),1).item()))\n    if ctr>1: break\n    ctr+=1","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:34:49.045948Z","iopub.execute_input":"2021-07-29T18:34:49.046530Z","iopub.status.idle":"2021-07-29T18:34:49.551936Z","shell.execute_reply.started":"2021-07-29T18:34:49.046475Z","shell.execute_reply":"2021-07-29T18:34:49.550967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pd.DataFrame(columns=['ImageId','Label'])\nprediction = []\nfor img in submission_dl:\n    prediction.append(int(torch.argmax(model(img[0].unsqueeze(0)),1).item()))\ndf_submission['Label'] = pd.Series(prediction)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:34:49.553227Z","iopub.execute_input":"2021-07-29T18:34:49.553616Z","iopub.status.idle":"2021-07-29T18:36:32.528155Z","shell.execute_reply.started":"2021-07-29T18:34:49.553579Z","shell.execute_reply":"2021-07-29T18:36:32.527214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission['ImageId'] = df_submission.Label.index + 1","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:36:32.529540Z","iopub.execute_input":"2021-07-29T18:36:32.529922Z","iopub.status.idle":"2021-07-29T18:36:32.539122Z","shell.execute_reply.started":"2021-07-29T18:36:32.529885Z","shell.execute_reply":"2021-07-29T18:36:32.538137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_submission.to_csv('submission_resnet.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:36:32.540509Z","iopub.execute_input":"2021-07-29T18:36:32.541018Z","iopub.status.idle":"2021-07-29T18:36:32.547637Z","shell.execute_reply.started":"2021-07-29T18:36:32.540978Z","shell.execute_reply":"2021-07-29T18:36:32.546856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#torch.save(model.state_dict(),'resnet_digits_model.pth')","metadata":{"execution":{"iopub.status.busy":"2021-07-29T18:36:32.548882Z","iopub.execute_input":"2021-07-29T18:36:32.549295Z","iopub.status.idle":"2021-07-29T18:36:32.557100Z","shell.execute_reply.started":"2021-07-29T18:36:32.549259Z","shell.execute_reply":"2021-07-29T18:36:32.556384Z"},"trusted":true},"execution_count":null,"outputs":[]}]}