{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install py7zr","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:41:07.048538Z","iopub.execute_input":"2024-05-28T05:41:07.049375Z","iopub.status.idle":"2024-05-28T05:41:07.054111Z","shell.execute_reply.started":"2024-05-28T05:41:07.049338Z","shell.execute_reply":"2024-05-28T05:41:07.053089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import py7zr \n\narchive=py7zr.SevenZipFile('../input/cifar-10/train.7z', mode='r')\narchive.extractall(path=\"/kaggle/temp\")\narchive.close()","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:19:19.429325Z","iopub.execute_input":"2024-05-28T05:19:19.430121Z","iopub.status.idle":"2024-05-28T05:19:48.565830Z","shell.execute_reply.started":"2024-05-28T05:19:19.430086Z","shell.execute_reply":"2024-05-28T05:19:48.564712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/cifar-10/trainLabels.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:14.060934Z","iopub.execute_input":"2024-05-28T05:20:14.061477Z","iopub.status.idle":"2024-05-28T05:20:14.099373Z","shell.execute_reply.started":"2024-05-28T05:20:14.061452Z","shell.execute_reply":"2024-05-28T05:20:14.098591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:14.100438Z","iopub.execute_input":"2024-05-28T05:20:14.100731Z","iopub.status.idle":"2024-05-28T05:20:14.119179Z","shell.execute_reply.started":"2024-05-28T05:20:14.100706Z","shell.execute_reply":"2024-05-28T05:20:14.118292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create folders with the names of classes\nos.mkdir(os.path.join('/kaggle/temp', 'dataset'))\nfor label in df['label'].unique():\n    print(label)\n    os.mkdir(os.path.join('/kaggle/temp/dataset', label))","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:14.127739Z","iopub.execute_input":"2024-05-28T05:20:14.128409Z","iopub.status.idle":"2024-05-28T05:20:14.143536Z","shell.execute_reply.started":"2024-05-28T05:20:14.128372Z","shell.execute_reply":"2024-05-28T05:20:14.142678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from shutil import copy2\n\n# copy images to the corresponding folders\nfor dirname, _, filenames in os.walk('/kaggle/temp'):\n    for filename in filenames:\n        r_path=os.path.join(dirname, filename)[::-1]\n        i = r_path[:r_path.index('/')]\n        i = i[::-1]\n        i = int(i[:i.index('.')])-1\n        label = df.iloc[i].label\n        copy2(os.path.join('/kaggle/temp/train', filename), \n              os.path.join(f'/kaggle/temp/dataset/{label}', filename))","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:14.174222Z","iopub.execute_input":"2024-05-28T05:20:14.174536Z","iopub.status.idle":"2024-05-28T05:20:27.548198Z","shell.execute_reply.started":"2024-05-28T05:20:14.174512Z","shell.execute_reply":"2024-05-28T05:20:27.547349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  TensorFlow Implementation","metadata":{}},{"cell_type":"code","source":"# The implemantation from TensorFlow tutorial\n# For more details: https://www.tensorflow.org/tutorials/images/classification#visualize_training_results","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:42:12.526431Z","iopub.execute_input":"2024-05-28T05:42:12.527214Z","iopub.status.idle":"2024-05-28T05:42:12.532110Z","shell.execute_reply.started":"2024-05-28T05:42:12.527177Z","shell.execute_reply":"2024-05-28T05:42:12.530880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\nimg_height = 32\nimg_width = 32","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:42:12.914579Z","iopub.execute_input":"2024-05-28T05:42:12.914967Z","iopub.status.idle":"2024-05-28T05:42:12.920601Z","shell.execute_reply.started":"2024-05-28T05:42:12.914936Z","shell.execute_reply":"2024-05-28T05:42:12.919500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pathlib\ndata_dir = pathlib.Path('/kaggle/temp/dataset').with_suffix('')","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:42:13.334908Z","iopub.execute_input":"2024-05-28T05:42:13.335813Z","iopub.status.idle":"2024-05-28T05:42:13.340351Z","shell.execute_reply.started":"2024-05-28T05:42:13.335776Z","shell.execute_reply":"2024-05-28T05:42:13.339482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = tf.keras.utils.image_dataset_from_directory(\n  data_dir,\n  validation_split=0.2,\n  subset=\"training\",\n  seed=123,\n  image_size=(img_height, img_width),\n  batch_size=batch_size\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:42:13.815483Z","iopub.execute_input":"2024-05-28T05:42:13.815869Z","iopub.status.idle":"2024-05-28T05:42:17.384102Z","shell.execute_reply.started":"2024-05-28T05:42:13.815838Z","shell.execute_reply":"2024-05-28T05:42:17.383070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_ds = tf.keras.utils.image_dataset_from_directory(\n  data_dir,\n  validation_split=0.2,\n  subset=\"validation\",\n  seed=123,\n  image_size=(img_height, img_width),\n  batch_size=batch_size\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:42:18.507235Z","iopub.execute_input":"2024-05-28T05:42:18.507643Z","iopub.status.idle":"2024-05-28T05:42:21.910878Z","shell.execute_reply.started":"2024-05-28T05:42:18.507612Z","shell.execute_reply":"2024-05-28T05:42:21.910044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names = train_ds.class_names","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:42:22.857623Z","iopub.execute_input":"2024-05-28T05:42:22.858401Z","iopub.status.idle":"2024-05-28T05:42:22.862675Z","shell.execute_reply.started":"2024-05-28T05:42:22.858356Z","shell.execute_reply":"2024-05-28T05:42:22.861620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at image of each class\nadded_labels = []\nadded_images = []\nplt.figure(figsize=(5, 5))\nfor image_batch, labels_batch in train_ds.take(1):\n    for img, label in zip(image_batch, labels_batch):\n        if label not in added_labels:\n            added_labels.append(label)\n            ax = plt.subplot(3, 5, len(added_labels))\n            plt.imshow(img.numpy().astype(\"uint8\"))\n            plt.title(class_names[label])\n            plt.axis(\"off\")\n   \n","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:38.609618Z","iopub.execute_input":"2024-05-28T05:20:38.609933Z","iopub.status.idle":"2024-05-28T05:20:39.692927Z","shell.execute_reply.started":"2024-05-28T05:20:38.609908Z","shell.execute_reply":"2024-05-28T05:20:39.692022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\ntrain_ds = train_ds.cache().shuffle(1000).prefetch(buffer_size=AUTOTUNE)\nval_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:39.694387Z","iopub.execute_input":"2024-05-28T05:20:39.695090Z","iopub.status.idle":"2024-05-28T05:20:39.712715Z","shell.execute_reply.started":"2024-05-28T05:20:39.695054Z","shell.execute_reply":"2024-05-28T05:20:39.711706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normalization_layer = layers.Rescaling(1./255)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:39.713797Z","iopub.execute_input":"2024-05-28T05:20:39.714107Z","iopub.status.idle":"2024-05-28T05:20:39.720721Z","shell.execute_reply.started":"2024-05-28T05:20:39.714080Z","shell.execute_reply":"2024-05-28T05:20:39.719802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normalized_ds = train_ds.map(lambda x, y: (normalization_layer(x), y))\nimage_batch, labels_batch = next(iter(normalized_ds))\nfirst_image = image_batch[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-28T06:22:42.523479Z","iopub.execute_input":"2024-05-28T06:22:42.524635Z","iopub.status.idle":"2024-05-28T06:22:42.599932Z","shell.execute_reply.started":"2024-05-28T06:22:42.524596Z","shell.execute_reply":"2024-05-28T06:22:42.598747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential([\n  layers.Rescaling(1./255, input_shape=(img_height, img_width, 3)),\n  layers.Conv2D(16, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Conv2D(32, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Conv2D(64, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Flatten(),\n  layers.Dense(128, activation='relu'),\n  layers.Dense(10)\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:42.250857Z","iopub.execute_input":"2024-05-28T05:20:42.251176Z","iopub.status.idle":"2024-05-28T05:20:42.360399Z","shell.execute_reply.started":"2024-05-28T05:20:42.251150Z","shell.execute_reply":"2024-05-28T05:20:42.359363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:42.361641Z","iopub.execute_input":"2024-05-28T05:20:42.361920Z","iopub.status.idle":"2024-05-28T05:20:42.375278Z","shell.execute_reply.started":"2024-05-28T05:20:42.361896Z","shell.execute_reply":"2024-05-28T05:20:42.373931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:42.376516Z","iopub.execute_input":"2024-05-28T05:20:42.376766Z","iopub.status.idle":"2024-05-28T05:20:42.402264Z","shell.execute_reply.started":"2024-05-28T05:20:42.376745Z","shell.execute_reply":"2024-05-28T05:20:42.401308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if GPU is available\ntf.config.list_physical_devices('GPU')","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:42.403625Z","iopub.execute_input":"2024-05-28T05:20:42.404329Z","iopub.status.idle":"2024-05-28T05:20:42.410764Z","shell.execute_reply.started":"2024-05-28T05:20:42.404294Z","shell.execute_reply":"2024-05-28T05:20:42.409814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs=10\nhistory = model.fit(\n  train_ds,\n  validation_data=val_ds,\n  epochs=epochs,\n  verbose=False\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:20:42.412001Z","iopub.execute_input":"2024-05-28T05:20:42.412404Z","iopub.status.idle":"2024-05-28T05:21:29.447588Z","shell.execute_reply.started":"2024-05-28T05:20:42.412369Z","shell.execute_reply":"2024-05-28T05:21:29.446512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(epochs)\n\nplt.figure(figsize=(8, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training Accuracy')\nplt.plot(epochs_range, val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:21:29.448900Z","iopub.execute_input":"2024-05-28T05:21:29.449213Z","iopub.status.idle":"2024-05-28T05:21:29.962319Z","shell.execute_reply.started":"2024-05-28T05:21:29.449186Z","shell.execute_reply":"2024-05-28T05:21:29.961192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A plot above indicates *overfitting*, since there's a big difference between our training error and cross-validation error. ","metadata":{}},{"cell_type":"markdown","source":"# Figting overfitting: data augmentation and dropout","metadata":{}},{"cell_type":"code","source":"data_augmentation = keras.Sequential(\n  [\n    layers.RandomFlip(\"horizontal\",\n                      input_shape=(img_height,\n                                  img_width,\n                                  3)),\n    layers.RandomRotation(0.1),\n    layers.RandomZoom(0.1),\n  ]\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:22:14.670298Z","iopub.execute_input":"2024-05-28T05:22:14.670678Z","iopub.status.idle":"2024-05-28T05:22:14.692531Z","shell.execute_reply.started":"2024-05-28T05:22:14.670649Z","shell.execute_reply":"2024-05-28T05:22:14.691730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5, 5))\nfor images, _ in train_ds.take(1):\n  for i in range(9):\n    augmented_images = data_augmentation(images)\n    ax = plt.subplot(3, 3, i + 1)\n    plt.imshow(augmented_images[0].numpy().astype(\"uint8\"))\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:27:36.067904Z","iopub.execute_input":"2024-05-28T05:27:36.068839Z","iopub.status.idle":"2024-05-28T05:27:36.598409Z","shell.execute_reply.started":"2024-05-28T05:27:36.068807Z","shell.execute_reply":"2024-05-28T05:27:36.597114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential([\n  data_augmentation,\n  layers.Rescaling(1./255),\n  layers.Conv2D(16, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Conv2D(32, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Conv2D(64, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Dropout(0.2),\n  layers.Flatten(),\n  layers.Dense(128, activation='relu'),\n  layers.Dense(10, name=\"outputs\")\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:28:38.787571Z","iopub.execute_input":"2024-05-28T05:28:38.788219Z","iopub.status.idle":"2024-05-28T05:28:38.807468Z","shell.execute_reply.started":"2024-05-28T05:28:38.788187Z","shell.execute_reply":"2024-05-28T05:28:38.806666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:28:47.608435Z","iopub.execute_input":"2024-05-28T05:28:47.609193Z","iopub.status.idle":"2024-05-28T05:28:47.618561Z","shell.execute_reply.started":"2024-05-28T05:28:47.609153Z","shell.execute_reply":"2024-05-28T05:28:47.617731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 15\nhistory = model.fit(\n  train_ds,\n  validation_data=val_ds,\n  epochs=epochs,\n  verbose=False\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T06:28:55.013608Z","iopub.execute_input":"2024-05-28T06:28:55.014566Z","iopub.status.idle":"2024-05-28T06:31:13.709876Z","shell.execute_reply.started":"2024-05-28T06:28:55.014529Z","shell.execute_reply":"2024-05-28T06:31:13.709047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(epochs)\n\nplt.figure(figsize=(8, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training Accuracy')\nplt.plot(epochs_range, val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-28T05:31:00.299779Z","iopub.execute_input":"2024-05-28T05:31:00.300202Z","iopub.status.idle":"2024-05-28T05:31:00.826460Z","shell.execute_reply.started":"2024-05-28T05:31:00.300169Z","shell.execute_reply":"2024-05-28T05:31:00.825417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training error and cross-validation error are almost the same and both are pretty low, so the procedure above was successful","metadata":{}},{"cell_type":"code","source":"# Look at predictions on validation set\nplt.figure(figsize=(5, 5))\nfor images, labels in val_ds.take(1):\n    yhat = model.predict(images)\n    for i in range(15):\n        ax = plt.subplot(3, 5, i + 1)\n        plt.imshow(images[i].numpy().astype(\"uint8\"))\n        plt.title(class_names[np.argmax(yhat[i])])\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2024-05-28T06:21:09.635124Z","iopub.execute_input":"2024-05-28T06:21:09.635537Z","iopub.status.idle":"2024-05-28T06:21:10.820857Z","shell.execute_reply.started":"2024-05-28T06:21:09.635505Z","shell.execute_reply":"2024-05-28T06:21:10.819813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  Look at the misclassiefied images","metadata":{}},{"cell_type":"code","source":"predictions = []\ny = []\nX = [] \nfor images, labels in train_ds:\n    yhat = model.predict(images, verbose=0)\n    predictions.extend(list(map(lambda y: np.argmax(y), yhat)))\n    y.extend(list(map(lambda label: np.argmax(label), labels)))\n    X.extend(images)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T06:07:24.286831Z","iopub.execute_input":"2024-05-28T06:07:24.287506Z","iopub.status.idle":"2024-05-28T06:09:08.910485Z","shell.execute_reply.started":"2024-05-28T06:07:24.287472Z","shell.execute_reply":"2024-05-28T06:09:08.909418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = np.array(predictions) \ny = np.array(y)\nX = np.array(X)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T06:10:35.615714Z","iopub.execute_input":"2024-05-28T06:10:35.616427Z","iopub.status.idle":"2024-05-28T06:10:37.689199Z","shell.execute_reply.started":"2024-05-28T06:10:35.616376Z","shell.execute_reply":"2024-05-28T06:10:37.688333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_missclassified = np.absolute(y-predictions).astype(bool)","metadata":{"execution":{"iopub.status.busy":"2024-05-28T06:12:41.748551Z","iopub.execute_input":"2024-05-28T06:12:41.748910Z","iopub.status.idle":"2024-05-28T06:12:41.754207Z","shell.execute_reply.started":"2024-05-28T06:12:41.748882Z","shell.execute_reply":"2024-05-28T06:12:41.753143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5, 5))\nk = 1\nfor i, image in enumerate(X[:50]):\n    if is_missclassified[i]:\n        yhat = model.predict(images, verbose=0)\n\n        ax = plt.subplot(3, 5, k % 15 + 1)\n        k += 1\n        plt.imshow(image.astype(\"uint8\"))\n        plt.title(class_names[np.argmax(predictions[i])])\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2024-05-28T06:20:29.567809Z","iopub.execute_input":"2024-05-28T06:20:29.568505Z","iopub.status.idle":"2024-05-28T06:20:33.635129Z","shell.execute_reply.started":"2024-05-28T06:20:29.568468Z","shell.execute_reply":"2024-05-28T06:20:33.634188Z"},"trusted":true},"execution_count":null,"outputs":[]}]}