{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torch.utils.data import ConcatDataset\nfrom torchvision import transforms\nfrom PIL import Image\nimport pytorch_lightning as pl\nimport torchvision\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:27:36.738355Z","iopub.execute_input":"2023-05-27T06:27:36.738781Z","iopub.status.idle":"2023-05-27T06:27:49.890824Z","shell.execute_reply.started":"2023-05-27T06:27:36.738739Z","shell.execute_reply":"2023-05-27T06:27:49.889915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:27:49.892770Z","iopub.execute_input":"2023-05-27T06:27:49.893141Z","iopub.status.idle":"2023-05-27T06:27:49.897569Z","shell.execute_reply.started":"2023-05-27T06:27:49.893108Z","shell.execute_reply":"2023-05-27T06:27:49.896633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nfrom collections import Counter\nfrom tqdm import tqdm\n# define directories\nIMAGE_DIR = '/kaggle/input/benetech-making-graphs-accessible/train/images'\nANNOTATIONS_DIR = '/kaggle/input/benetech-making-graphs-accessible/train/annotations'\n\n# get the list of images\nimage_filenames = os.listdir(IMAGE_DIR)\n\njson_files = json.load(open(\"/kaggle/input/annotation-data-whole/data.json\"))","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:28:20.306658Z","iopub.execute_input":"2023-05-27T06:28:20.307035Z","iopub.status.idle":"2023-05-27T06:28:40.737027Z","shell.execute_reply.started":"2023-05-27T06:28:20.307002Z","shell.execute_reply":"2023-05-27T06:28:40.736080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import seaborn as sns\n# import matplotlib.pyplot as plt\n# import numpy as np\n\n# # Data\n# y = ['Alberta','Ontario','Quebec','P.E.I.']\n# x = [62023,50355,56288,38621]\n\n# # Create a pandas DataFrame\n# df = pd.DataFrame(list(zip(y, x)), columns =['Province', 'Value']) \n\n# # Define list of themes and colors\n# themes = ['darkgrid', 'whitegrid', 'dark', 'white', 'ticks']\n# colors = ['blue', 'green', 'red', 'cyan', 'magenta', 'yellow', 'black', 'purple', 'pink', 'brown', 'orange', 'teal', 'coral', 'lightblue', 'lime', 'lavender', 'turquoise', 'darkgreen', 'tan', 'salmon', 'gold', 'lightpurple', 'darkred', 'darkblue']\n\n# # Select random theme and color\n# theme = np.random.choice(themes)\n# color = np.random.choice(colors)\n\n# # Set the theme\n# sns.set_style(theme)\n\n# # Create the bar plot\n# plt.figure(figsize=(7,5))\n# ax = sns.barplot(y=\"Province\", x=\"Value\", data=df, palette=[color])\n# plt.title('Random Bar Plot')\n\n# # Save the figure\n# plt.savefig(\"barplot.png\", dpi=70)\n\n# # Display the plot\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:28:40.738763Z","iopub.execute_input":"2023-05-27T06:28:40.739126Z","iopub.status.idle":"2023-05-27T06:28:40.745342Z","shell.execute_reply.started":"2023-05-27T06:28:40.739091Z","shell.execute_reply":"2023-05-27T06:28:40.744291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = {}\nfor imgf in tqdm(os.listdir(IMAGE_DIR)):\n    img_path = os.path.join(IMAGE_DIR, imgf)\n    base_filename = os.path.splitext(imgf)[0]\n    images[base_filename] = img_path","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:28:40.746560Z","iopub.execute_input":"2023-05-27T06:28:40.747495Z","iopub.status.idle":"2023-05-27T06:28:41.085746Z","shell.execute_reply.started":"2023-05-27T06:28:40.747461Z","shell.execute_reply":"2023-05-27T06:28:41.084697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame([[x[\"chart-type\"],x[\"source\"],k] for k,x in json_files.items()])\nprint(df[1].value_counts())\nprint(\"----------------------------------------\")\nprint(df[0].value_counts())\nprint(\"----------------------------------------\")\nprint(df[df[1]==\"extracted\"][0].value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:28:41.088680Z","iopub.execute_input":"2023-05-27T06:28:41.089129Z","iopub.status.idle":"2023-05-27T06:28:41.217349Z","shell.execute_reply.started":"2023-05-27T06:28:41.089091Z","shell.execute_reply":"2023-05-27T06:28:41.216400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image_and_chart_type(image_path, annotation_path):\n    image = Image.open(image_path)\n    with open(annotation_path, 'r') as f:\n        annotation = json.load(f)\n    return image, annotation['chart-type']\n\n# Select 8 random images\nannotations_file = random.sample(df[2][df[1]==\"extracted\"].tolist(), 8)\n\nplt.figure(figsize=(20, 20))\nfor i, base_filename in enumerate(annotations_file, start=1):\n    annotation_filename = base_filename + '.json'\n    annotation_path = os.path.join(ANNOTATIONS_DIR, annotation_filename)\n    image_path = images[base_filename]\n    image, chart_type = load_image_and_chart_type(image_path, annotation_path)\n    \n    plt.subplot(4, 2, i)\n    plt.imshow(image)\n    plt.title(\"Extracted \"+ chart_type)\n    plt.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:28:41.218802Z","iopub.execute_input":"2023-05-27T06:28:41.219404Z","iopub.status.idle":"2023-05-27T06:28:42.714193Z","shell.execute_reply.started":"2023-05-27T06:28:41.219367Z","shell.execute_reply":"2023-05-27T06:28:42.713098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image_and_chart_type(image_path, annotation_path):\n    image = Image.open(image_path)\n    with open(annotation_path, 'r') as f:\n        annotation = json.load(f)\n    return image, annotation['chart-type']\n\n# Select 8 random images\nannotations_file = random.sample(df[2][df[1]==\"generated\"].tolist(), 8)\n\nplt.figure(figsize=(20, 20))\nfor i, base_filename in enumerate(annotations_file, start=1):\n    annotation_filename = base_filename + '.json'\n    annotation_path = os.path.join(ANNOTATIONS_DIR, annotation_filename)\n    image_path = images[base_filename]\n    image, chart_type = load_image_and_chart_type(image_path, annotation_path)\n    \n    plt.subplot(4, 2, i)\n    plt.imshow(image)\n    plt.title(\"generated \"+ chart_type)\n    plt.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:28:42.715165Z","iopub.execute_input":"2023-05-27T06:28:42.715476Z","iopub.status.idle":"2023-05-27T06:28:44.347042Z","shell.execute_reply.started":"2023-05-27T06:28:42.715448Z","shell.execute_reply":"2023-05-27T06:28:44.344006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Synthetic horizontal_bar images","metadata":{}},{"cell_type":"markdown","source":"##### We can see that the number of  horizontal_bar images is very small: 73. So, lets add some synthetic horizontal_bar images to the dataset. To do this, we can randomly rotate vertical_bar images clockwise by 90. The signal provided by these rotated images might help the model to learn what a horizontal_bar should look like.","metadata":{}},{"cell_type":"code","source":"def load_image_and_chart_type(image_path, annotation_path):\n    image = Image.open(image_path)\n    with open(annotation_path, 'r') as f:\n        annotation = json.load(f)\n    return image, annotation['chart-type']\n\n# Select 8 random images\nannotations_file = random.sample(df[2][df[0]==\"horizontal_bar\"].tolist(), 8)\n\nplt.figure(figsize=(10, 10))\nfor i, base_filename in enumerate(annotations_file, start=1):\n    annotation_filename = base_filename + '.json'\n    annotation_path = os.path.join(ANNOTATIONS_DIR, annotation_filename)\n    image_path = images[base_filename]\n    image, chart_type = load_image_and_chart_type(image_path, annotation_path)\n    \n    plt.subplot(4, 2, i)\n    plt.imshow(image)\n    plt.title(\"Extracted \"+ chart_type)\n    plt.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:28:44.348414Z","iopub.execute_input":"2023-05-27T06:28:44.348950Z","iopub.status.idle":"2023-05-27T06:28:45.260496Z","shell.execute_reply.started":"2023-05-27T06:28:44.348904Z","shell.execute_reply":"2023-05-27T06:28:45.259583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Select 8 random images\nannotations_file = random.sample(df[2][df[0]==\"vertical_bar\"].tolist(), 8)\n\nplt.figure(figsize=(10, 10))\nfor i, base_filename in enumerate(annotations_file, start=1):\n    annotation_filename = base_filename + '.json'\n    annotation_path = os.path.join(ANNOTATIONS_DIR, annotation_filename)\n    image_path = images[base_filename]\n    image, chart_type = load_image_and_chart_type(image_path, annotation_path)\n    \n    plt.subplot(4, 2, i)\n    plt.imshow(image.rotate(-90,expand=True))\n    plt.title(\"Generated Horizontal Bar\")\n    plt.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:28:45.261865Z","iopub.execute_input":"2023-05-27T06:28:45.262499Z","iopub.status.idle":"2023-05-27T06:28:46.080765Z","shell.execute_reply.started":"2023-05-27T06:28:45.262460Z","shell.execute_reply":"2023-05-27T06:28:46.079998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training and Evaluation","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\ndef plot_confusion_matrix(cm, class_names,title):\n    plt.figure(figsize=(5, 5))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', cbar=False, xticklabels=class_names, yticklabels=class_names)\n    plt.xlabel('Predicted')\n    plt.ylabel('Actual')\n    plt.title(title)\n    plt.show()\n\ndef pil2cv(image):\n    ''' PIL型 -> OpenCV型 '''\n    return cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:29:07.215443Z","iopub.execute_input":"2023-05-27T06:29:07.215810Z","iopub.status.idle":"2023-05-27T06:29:07.223257Z","shell.execute_reply.started":"2023-05-27T06:29:07.215779Z","shell.execute_reply":"2023-05-27T06:29:07.221988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ChartDataset(Dataset):\n    def __init__(self, images, annotations,transform=None,train_transform=None):\n        super().__init__()\n        self.images = images\n        self.annotations = annotations\n        self.transform = transform\n        self.train_transform = train_transform\n        self.keys = list(self.annotations.keys())\n        self.class_map = {'line': 0, 'dot': 1, 'vertical_bar': 2, 'scatter': 3, 'horizontal_bar': 4}\n        self.is_train=False\n        self.create_horizontal = False\n    def _check_source(self, img_file, source):\n        key = os.path.splitext(img_file)[0]\n        ann = self.annotations[key]\n        return ann['source'] == source\n    \n    def __len__(self):\n        return len(self.keys)\n    def set_train(self):\n        self.is_train=True\n    def __getitem__(self, idx):\n        key = self.keys[idx]\n        img_path = self.images[key]\n        image = Image.open(img_path)\n        ann = self.annotations[key]\n        charttype=ann['chart-type']\n        if self.create_horizontal and charttype==\"vertical_bar\" and np.random.rand()<0.2:\n            image = image.rotate(-90,expand=True)\n            charttype= \"horizontal_bar\"\n        image = pil2cv(image)\n        if self.is_train:\n            image = self.train_transform(image=image)[\"image\"]\n        else:\n            image = self.transform(image=image)[\"image\"]\n        return image, self.class_map[charttype],ann['source']","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:36:12.012634Z","iopub.execute_input":"2023-05-27T06:36:12.013017Z","iopub.status.idle":"2023-05-27T06:36:12.024892Z","shell.execute_reply.started":"2023-05-27T06:36:12.012983Z","shell.execute_reply":"2023-05-27T06:36:12.023860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ChartModel(pl.LightningModule):\n    def __init__(self, num_classes):\n        super().__init__()\n        self.num_classes = num_classes\n        self.class_names = ['line', 'dot', 'vertical_bar', 'scatter', 'horizontal_bar']\n        self.save_hyperparameters()\n        self.efficientnet = torchvision.models.efficientnet_b2(torchvision.models.EfficientNet_B2_Weights.DEFAULT)\n        self.lr = 3e-4\n        self.efficientnet.classifier[1] = torch.nn.Linear(self.efficientnet.classifier[1].in_features, num_classes)\n        self.loss = torch.nn.CrossEntropyLoss()\n        self.predictions = []\n        self.targets = []\n        self.sources = []\n        self.best_val_loss=9999\n        self.epochs_since_best_loss =0\n        self.unfrozen_after = 7\n        self.unfreeze_layer()\n    def forward(self, x):\n        return self.efficientnet(x)\n\n    def training_step(self, batch, batch_idx):\n        x, y,_ = batch\n        y_hat = self(x)\n        loss = self.loss(y_hat, y)\n        self.log('train_loss', loss)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        x, y,_ = batch\n        y_hat = self(x)\n        loss = self.loss(y_hat, y)\n        self.log('val_loss', loss)\n        return loss\n\n    def test_step(self, batch, batch_idx):\n        x, y, source = batch\n        source = np.array(source)\n        y_hat = self(x)\n        loss = self.loss(y_hat, y)\n        self.log('test_loss', loss)\n        \n        # log metrics for different sources separately\n        preds = torch.argmax(y_hat, dim=1)\n        \n        self.predictions.append(preds)\n        self.targets.append(y)\n        self.sources.append(source)\n    def on_test_epoch_end(self):\n        preds = torch.cat(self.predictions).cpu().numpy()\n        targets = torch.cat(self.targets).cpu().numpy()\n        sources = np.concatenate(self.sources)\n        print(\"accuracy: \",accuracy_score(targets,preds))\n        if (sources==\"generated\").sum()>0:\n            cm = confusion_matrix(targets[sources==\"generated\"], preds[sources==\"generated\"],labels=range(self.num_classes))\n            plot_confusion_matrix(cm, self.class_names,\"generated\")\n        if (sources==\"extracted\").sum()>0:\n            cm = confusion_matrix(targets[sources==\"extracted\"], preds[sources==\"extracted\"],labels=range(self.num_classes))\n            plot_confusion_matrix(cm, self.class_names,\"extracted\")\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(),lr=self.lr)\n    \n    def on_validation_epoch_end(self):\n        current_val_loss = self.trainer.callback_metrics['val_loss']\n        if current_val_loss < self.best_val_loss:\n            self.best_val_loss = current_val_loss\n            self.epochs_since_best_loss = 0\n        else:\n            self.epochs_since_best_loss += 1\n\n        if self.epochs_since_best_loss >= 5:\n            self.unfrozen_after -=1\n            self.unfrozen_after = max(self.unfrozen_after,0)\n            print(\"Unfreezing layer: \",self.unfrozen_after)\n            \n            self.unfreeze_layer()\n            self.lr /= 2\n            self.epochs_since_best_loss = 0\n            self.trainer.optimizers[0] = self.configure_optimizers()\n            \n    def unfreeze_layer(self):\n        reqgrad=False\n        for name,param in self.efficientnet.named_parameters():\n            if f\"features.{self.unfrozen_after}\" in name:\n                reqgrad = True\n            param.requires_grad = reqgrad","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:36:13.665348Z","iopub.execute_input":"2023-05-27T06:36:13.665710Z","iopub.status.idle":"2023-05-27T06:36:13.693808Z","shell.execute_reply.started":"2023-05-27T06:36:13.665677Z","shell.execute_reply":"2023-05-27T06:36:13.692826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:36:14.817616Z","iopub.execute_input":"2023-05-27T06:36:14.818688Z","iopub.status.idle":"2023-05-27T06:36:14.823746Z","shell.execute_reply.started":"2023-05-27T06:36:14.818643Z","shell.execute_reply":"2023-05-27T06:36:14.822859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for name,param in model.efficientnet.named_parameters():\n#     if param.requires_grad:\n#         print(name)","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:36:15.038517Z","iopub.execute_input":"2023-05-27T06:36:15.039359Z","iopub.status.idle":"2023-05-27T06:36:15.048263Z","shell.execute_reply.started":"2023-05-27T06:36:15.039318Z","shell.execute_reply":"2023-05-27T06:36:15.047372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nskf = StratifiedKFold(n_splits=5)","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:36:15.335904Z","iopub.execute_input":"2023-05-27T06:36:15.336560Z","iopub.status.idle":"2023-05-27T06:36:15.340864Z","shell.execute_reply.started":"2023-05-27T06:36:15.336526Z","shell.execute_reply":"2023-05-27T06:36:15.339976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUGMENTATIONS = [\n                    A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1, always_apply=False, p=0.5),\n                    A.OneOf([\n                        A.GaussNoise(var_limit=(10.0, 20.0), mean=0, per_channel=True, always_apply=False, p=0.5),\n                        A.ISONoise(color_shift=(0.01, 0.05), intensity=(0.1, 0.5), always_apply=False, p=0.5),\n                    ], p=0.5),\n                    A.PixelDropout(dropout_prob=0.01, drop_value=None, always_apply=False, p=0.5),\n                    A.ToGray(p=0.5),\n                    A.RandomScale(scale_limit=(-0.5,0)),\n                    A.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, p=0.5),\n                    A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n                    A.OneOf([\n                        A.MotionBlur(blur_limit=5, p=0.5),\n                        A.MedianBlur(blur_limit=5, p=0.5),\n                    ], p=0.5),\n]\nALWAYS_APPLY = [\n                    A.Resize(288,288,p=1.0),\n                    A.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225),p=1.0),                  \n                    ToTensorV2()\n]","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:36:18.813691Z","iopub.execute_input":"2023-05-27T06:36:18.814081Z","iopub.status.idle":"2023-05-27T06:36:18.824436Z","shell.execute_reply.started":"2023-05-27T06:36:18.814048Z","shell.execute_reply":"2023-05-27T06:36:18.823436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [v[\"chart-type\"] for v in json_files.values()]\nkeys = [k for k in json_files]","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:39:13.392614Z","iopub.execute_input":"2023-05-27T06:39:13.393013Z","iopub.status.idle":"2023-05-27T06:39:13.524291Z","shell.execute_reply.started":"2023-05-27T06:39:13.392973Z","shell.execute_reply":"2023-05-27T06:39:13.523312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For each split, save the file names in the corresponding dictionary\nfor i, (train_index, test_index) in enumerate(skf.split(keys, labels)):\n    print(\"Fold: \",i)\n    train_json = {keys[k]:json_files[keys[k]] for k in train_index}\n    valid_json = {keys[k]:json_files[keys[k]] for k in test_index}    \n    train_set = ChartDataset(images, train_json, transform=A.Compose(ALWAYS_APPLY),train_transform=A.Compose(AUGMENTATIONS+ALWAYS_APPLY))\n    train_set.set_train()\n    train_set.create_horizontal = True\n\n    valid_set = ChartDataset(images, valid_json, transform=A.Compose(ALWAYS_APPLY),train_transform=A.Compose(AUGMENTATIONS+ALWAYS_APPLY))\n\n    print(len(train_set),len(valid_set))\n    train_loader = DataLoader(train_set, batch_size=32, shuffle=True,num_workers=2)\n    valid_loader = DataLoader(valid_set, batch_size=32)\n\n    num_classes = 5  # update this based on your dataset\n    model = ChartModel(num_classes)\n\n    checkpoint_callback = ModelCheckpoint(\n        monitor='val_loss',\n        dirpath='checkpoints',\n        filename=f\"fold{i}_\"+'chart-model-{epoch:02d}-{val_loss:.2f}',\n        save_top_k=3,\n        mode='min',\n    )\n\n    early_stop_callback = EarlyStopping(\n        monitor='val_loss',\n        patience=10,\n        verbose=True,\n        mode='min'\n    )\n\n    trainer = pl.Trainer(max_epochs=5, callbacks=[checkpoint_callback, early_stop_callback])\n    trainer.fit(model, train_loader, valid_loader)\n    # Load the best model\n    model.load_from_checkpoint(checkpoint_callback.best_model_path)\n\n    trainer.test(model, valid_loader)","metadata":{"execution":{"iopub.status.busy":"2023-05-27T06:37:28.034960Z","iopub.execute_input":"2023-05-27T06:37:28.035359Z","iopub.status.idle":"2023-05-27T06:38:25.420922Z","shell.execute_reply.started":"2023-05-27T06:37:28.035328Z","shell.execute_reply":"2023-05-27T06:38:25.419316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}