{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2022-11-25T02:08:47.151508Z","iopub.execute_input":"2022-11-25T02:08:47.152269Z","iopub.status.idle":"2022-11-25T02:08:52.000607Z","shell.execute_reply.started":"2022-11-25T02:08:47.152164Z","shell.execute_reply":"2022-11-25T02:08:51.999564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" print(tf. __version__)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T02:09:07.964771Z","iopub.execute_input":"2022-11-25T02:09:07.965387Z","iopub.status.idle":"2022-11-25T02:09:07.971227Z","shell.execute_reply.started":"2022-11-25T02:09:07.96535Z","shell.execute_reply":"2022-11-25T02:09:07.970182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -V","metadata":{"execution":{"iopub.status.busy":"2022-11-25T02:11:06.470876Z","iopub.execute_input":"2022-11-25T02:11:06.471433Z","iopub.status.idle":"2022-11-25T02:11:07.554854Z","shell.execute_reply.started":"2022-11-25T02:11:06.471389Z","shell.execute_reply":"2022-11-25T02:11:07.553691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import platform\nprint(platform.system())","metadata":{"execution":{"iopub.status.busy":"2022-11-26T04:14:16.374845Z","iopub.execute_input":"2022-11-26T04:14:16.375827Z","iopub.status.idle":"2022-11-26T04:14:16.397829Z","shell.execute_reply.started":"2022-11-26T04:14:16.375727Z","shell.execute_reply":"2022-11-26T04:14:16.396837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip list","metadata":{"execution":{"iopub.status.busy":"2022-11-26T02:20:36.119287Z","iopub.execute_input":"2022-11-26T02:20:36.119717Z","iopub.status.idle":"2022-11-26T02:20:40.14524Z","shell.execute_reply.started":"2022-11-26T02:20:36.119634Z","shell.execute_reply":"2022-11-26T02:20:40.144059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport skimage\nprint('matplotlib: {}'.format(skimage.__version__))","metadata":{"execution":{"iopub.status.busy":"2022-11-25T04:04:05.227436Z","iopub.execute_input":"2022-11-25T04:04:05.228517Z","iopub.status.idle":"2022-11-25T04:04:05.249964Z","shell.execute_reply.started":"2022-11-25T04:04:05.228464Z","shell.execute_reply":"2022-11-25T04:04:05.248623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport numpy as np\nfrom tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.layers import Input, UpSampling2D, Flatten, BatchNormalization, Dense, Dropout, GlobalAveragePooling2D , UpSampling3D , Lambda\nfrom tensorflow.keras import optimizers\nfrom keras.datasets import cifar100,cifar10,fashion_mnist, mnist\nimport tensorflow as tf\nfrom keras.utils import np_utils\nimport matplotlib.pyplot as plt\nimport time\nfrom skimage.transform import resize\nfrom keras import applications\nfrom tensorflow.keras.applications.resnet50 import ResNet50 , preprocess_input , decode_predictions\nfrom keras.preprocessing.image import ImageDataGenerator\ntf.keras.backend.clear_session()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:58:29.527611Z","iopub.execute_input":"2022-12-12T07:58:29.528043Z","iopub.status.idle":"2022-12-12T07:58:30.776647Z","shell.execute_reply.started":"2022-12-12T07:58:29.528006Z","shell.execute_reply":"2022-12-12T07:58:30.775636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Distiller(keras.Model):\n    def __init__(self, student, teacher):\n        super(Distiller, self).__init__()\n        self.teacher = teacher\n        self.student = student\n\n    def compile(\n        self,\n        optimizer,\n        metrics,\n        student_loss_fn,\n        distillation_loss_fn,\n        alpha=0.2,\n        temperature=20,\n    ):\n        \"\"\" Configure the distiller.\n\n        Args:\n            optimizer: Keras optimizer for the student weights\n            metrics: Keras metrics for evaluation\n            student_loss_fn: Loss function of difference between student\n                predictions and ground-truth\n            distillation_loss_fn: Loss function of difference between soft\n                student predictions and soft teacher predictions\n            alpha: weight to student_loss_fn and 1-alpha to distillation_loss_fn\n            temperature: Temperature for softening probability distributions.\n                Larger temperature gives softer distributions.\n        \"\"\"\n        super(Distiller, self).compile(optimizer=optimizer, metrics=metrics)\n        self.student_loss_fn = student_loss_fn\n        self.distillation_loss_fn = distillation_loss_fn\n        self.alpha = alpha\n        self.temperature = temperature\n\n    def train_step(self, data):\n        # Unpack data\n        x, y = data\n\n        # Forward pass of teacher\n        teacher_predictions = self.teacher(x, training=False)\n\n        with tf.GradientTape() as tape:\n            # Forward pass of student\n            student_predictions = self.student(x, training=True)\n\n            # Compute losses\n            student_loss = self.student_loss_fn(y, student_predictions)\n            distillation_loss = self.distillation_loss_fn(\n                tf.nn.softmax(teacher_predictions / self.temperature, axis=1),\n                tf.nn.softmax(student_predictions / self.temperature, axis=1),\n            )\n            loss = self.alpha * student_loss + (1 - self.alpha) * distillation_loss\n\n        # Compute gradients\n        trainable_vars = self.student.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n\n        # Update weights\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n\n        # Update the metrics configured in `compile()`.\n        self.compiled_metrics.update_state(y, student_predictions)\n\n        # Return a dict of performance\n        results = {m.name: m.result() for m in self.metrics}\n        results.update(\n            {\"student_loss\": student_loss, \"distillation_loss\": distillation_loss}\n        )\n        return results\n\n    def test_step(self, data):\n        # Unpack the data\n        x, y = data\n\n        # Compute predictions\n        y_prediction = self.student(x, training=False)\n\n        # Calculate the loss\n        student_loss = self.student_loss_fn(y, y_prediction)\n\n        # Update the metrics.\n        self.compiled_metrics.update_state(y, y_prediction)\n\n        # Return a dict of performance\n        results = {m.name: m.result() for m in self.metrics}\n        results.update({\"student_loss\": student_loss})\n        return results\n\n    def call(self, inputs, *args, **kwargs):\n       return self.student(inputs)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:58:33.553893Z","iopub.execute_input":"2022-12-12T07:58:33.554299Z","iopub.status.idle":"2022-12-12T07:58:33.571876Z","shell.execute_reply.started":"2022-12-12T07:58:33.55426Z","shell.execute_reply":"2022-12-12T07:58:33.570946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_random_eraser(p=0.5, s_l=0.02, s_h=0.4, r_1=0.3, r_2=1/0.3, v_l=0, v_h=255, pixel_level=False):\n    def eraser(input_img):\n        img_h, img_w, img_c = input_img.shape\n        p_1 = np.random.rand()\n\n        if p_1 > p:\n            return input_img\n\n        while True:\n            s = np.random.uniform(s_l, s_h) * img_h * img_w\n            r = np.random.uniform(r_1, r_2)\n            w = int(np.sqrt(s / r))\n            h = int(np.sqrt(s * r))\n            left = np.random.randint(0, img_w)\n            top = np.random.randint(0, img_h)\n\n            if left + w <= img_w and top + h <= img_h:\n                break\n\n        if pixel_level:\n            c = np.random.uniform(v_l, v_h, (h, w, img_c))\n        else:\n            c = np.random.uniform(v_l, v_h)\n\n        input_img[top:top + h, left:left + w, :] = c\n\n        return input_img\n\n    return eraser","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:58:36.878226Z","iopub.execute_input":"2022-12-12T07:58:36.878587Z","iopub.status.idle":"2022-12-12T07:58:36.887465Z","shell.execute_reply.started":"2022-12-12T07:58:36.878555Z","shell.execute_reply":"2022-12-12T07:58:36.88633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def select_dataset(index=1): #1 for cifar10 , 2 for cifar100 , 3 for fashion mnist\n  if(index == 1):\n    (x_train, y_train), (x_test, y_test) = cifar10.load_data()\n    channel = 3\n    num_classes = 10\n  if(index == 2):\n    (x_train, y_train), (x_test, y_test) = cifar100.load_data()\n    num_classes = 100\n    channel = 3\n  if(index == 3):\n    (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\n    x_train = x_train.reshape((60000, 28, 28, 1))\n    x_test =  x_test.reshape((10000, 28, 28, 1))\n    num_classes = 10\n    y_train = np_utils.to_categorical(y_train, num_classes)\n    y_test = np_utils.to_categorical(y_test, num_classes)\n    channel = 1\n#   if(index == 4):\n#     (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n#     assert x_train.shape == (60000, 28, 28)\n#     assert x_test.shape == (10000, 28, 28)\n#     assert y_train.shape == (60000,)\n#     assert y_test.shape == (10000,)\n    \n    return (x_train , y_train , x_test , y_test , num_classes , channel)\n\n  #Pre-process the data\n  x_train = preprocess_input(x_train)\n  x_test = preprocess_input(x_test)\n\n  datagen = ImageDataGenerator(preprocessing_function=get_random_eraser(v_l=0, v_h=1, pixel_level=True))\n  datagen.fit(x_train)\n  y_train = np_utils.to_categorical(y_train, num_classes)\n  y_test = np_utils.to_categorical(y_test, num_classes)\n\n  return (x_train , y_train , x_test , y_test , num_classes , channel , datagen)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:58:39.022953Z","iopub.execute_input":"2022-12-12T07:58:39.023311Z","iopub.status.idle":"2022-12-12T07:58:39.033059Z","shell.execute_reply.started":"2022-12-12T07:58:39.02328Z","shell.execute_reply":"2022-12-12T07:58:39.031997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = 1\nx_train , y_train , x_test , y_test , num_classes ,channel , datagen = select_dataset(index)\nmodel_name = 'desktop/Trained_models/defensive_distillation_keras_resnet50_cifar10'\nmodel_path = 'desktop/Trained_models/defensive_distillation_keras_resnet50_cifar10.h5'","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:58:48.991174Z","iopub.execute_input":"2022-12-12T07:58:48.991529Z","iopub.status.idle":"2022-12-12T07:58:54.962523Z","shell.execute_reply.started":"2022-12-12T07:58:48.991498Z","shell.execute_reply":"2022-12-12T07:58:54.961493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = 200\nresnet_model = ResNet50(weights='imagenet', include_top=False, input_shape=(224, 224, 3))\nfor layer in resnet_model.layers:\n    if isinstance(layer, BatchNormalization):\n      layer.trainable = True\n        \n    else:\n      layer.trainable = False\nif(index==1):\n  teacher = keras.Sequential(\n      [\n        layers.UpSampling2D((7,7)),\n        resnet_model,\n        layers.GlobalAveragePooling2D(),\n        layers.Dense(256),\n        layers.ReLU(),\n        layers.Dropout(0.25),\n        layers.BatchNormalization(),\n        layers.Dense(32),\n        layers.ReLU(),\n        layers.Dropout(0.1),\n        layers.BatchNormalization(),\n        layers.Dense(num_classes),\n        layers.Lambda((lambda x: x / temp)),\n      ],\n      name=\"teacher\",\n  )\n\n# Create the student\n  student = keras.Sequential(\n      [\n        layers.UpSampling2D((7,7)),\n        resnet_model,\n        layers.GlobalAveragePooling2D(),\n        layers.Dense(256),\n        layers.ReLU(),\n        layers.Dropout(0.25),\n        layers.BatchNormalization(),\n        layers.Dense(32),\n        layers.ReLU(),\n        layers.Dropout(0.1),\n        layers.BatchNormalization(),\n        layers.Dense(num_classes),\n        layers.Lambda((lambda x: x / temp)),\n      ],\n      name=\"student\",\n  )\nelse:\n  teacher = keras.Sequential(\n      [\n        layers.UpSampling2D((7,7)),\n        resnet_model,\n        layers.GlobalAveragePooling2D(),\n        layers.Dense(1024),\n        layers.ReLU(),\n        layers.Dropout(0.3),\n        layers.BatchNormalization(),\n        layers.Dense(512),\n        layers.ReLU(),\n        layers.BatchNormalization(),\n        layers.Dense(256),\n        layers.ReLU(),\n        layers.BatchNormalization(),\n        layers.Dense(num_classes),\n        layers.Lambda((lambda x: x / temp)),\n      ],\n      name=\"teacher\",\n  )\n\n# Create the student\n  student = keras.Sequential(\n      [\n        layers.UpSampling2D((7,7)),\n        resnet_model,\n        layers.GlobalAveragePooling2D(),\n        layers.Dense(1024),\n        layers.ReLU(),\n        layers.Dropout(0.3),\n        layers.BatchNormalization(),\n        layers.Dense(512),\n        layers.ReLU(),\n        layers.BatchNormalization(),\n        layers.Dense(256),\n        layers.ReLU(),\n        layers.BatchNormalization(),\n        layers.Dense(num_classes),\n        layers.Lambda((lambda x: x / temp)),\n      ],\n      name=\"student\",\n  )\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-10T11:39:19.842101Z","iopub.execute_input":"2022-12-10T11:39:19.843074Z","iopub.status.idle":"2022-12-10T11:39:21.31044Z","shell.execute_reply.started":"2022-12-10T11:39:19.843036Z","shell.execute_reply":"2022-12-10T11:39:21.309468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 2  \ncallbacks = [ \n    tf.keras.callbacks.ModelCheckpoint(model_path, monitor='val_accuracy', verbose=1, save_best_only=True),\n    tf.keras.callbacks.ReduceLROnPlateau( factor = 0.7, patience = 5, min_lr = 0.000001, verbose = 1 ), #patience = 7 and 20 for cifar-100 , patience = 5 and 10 for cifar-10\n    #tf.keras.callbacks.EarlyStopping(monitor = 'val_accuracy' , patience = 10)\n  ]\nbatch_size = 128\nteacher.compile(\n    optimizer=keras.optimizers.Adam(),\n    loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n    metrics=[keras.metrics.CategoricalAccuracy()],\n)\n\n# Train and evaluate teacher on data.\nhistory_teacher = teacher.fit_generator(datagen.flow(x_train, y_train,\n                                  batch_size=batch_size),\n                                  steps_per_epoch=x_train.shape[0] // batch_size,\n                                  epochs=epochs,\n                                  validation_data=(x_test, y_test))\nteacher.evaluate(x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T11:39:54.129917Z","iopub.execute_input":"2022-12-10T11:39:54.130269Z","iopub.status.idle":"2022-12-10T11:46:59.690148Z","shell.execute_reply.started":"2022-12-10T11:39:54.130237Z","shell.execute_reply":"2022-12-10T11:46:59.689135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_format=\"/kaggle/working/desktop/history_teacher.h5\"\n# history_teacher.save('/kaggle/working/desktop/history_teacher.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-10T12:15:02.428732Z","iopub.execute_input":"2022-12-10T12:15:02.429092Z","iopub.status.idle":"2022-12-10T12:15:02.433344Z","shell.execute_reply.started":"2022-12-10T12:15:02.429061Z","shell.execute_reply":"2022-12-10T12:15:02.43233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import regularizers\ntemp = 200\nweight_decay = 0.0005\nif(index==1 or index==2):\n  teacher = keras.Sequential(\n      [\n        layers.UpSampling2D((5,5)),\n        layers.Conv2D(64,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay),input_shape = (160,160,3)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(64,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Conv2D(128,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(128,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Conv2D(256,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(256,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(256,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Flatten(),\n        layers.Dense(512,kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Dropout(0.25),\n        layers.Dense(num_classes),\n        layers.BatchNormalization(),\n        layers.Lambda((lambda x: x / temp)),\n      ],\n      name=\"teacher\",\n  )\n\n# Create the student\n  student = keras.Sequential(\n      [\n        layers.UpSampling2D((5,5)),\n        layers.Conv2D(64,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay),input_shape = (160,160,3)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(64,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Conv2D(128,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(128,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Conv2D(256,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(256,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(256,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Conv2D(512,3,padding=\"same\",kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.MaxPooling2D((2,2)),\n        layers.Dropout(0.25),\n        layers.Flatten(),\n        layers.Dense(512,kernel_regularizer=regularizers.l2(weight_decay)),\n        layers.BatchNormalization(),\n        layers.ReLU(),\n        layers.Dropout(0.25),\n        layers.Dense(num_classes),\n        layers.BatchNormalization(),\n        layers.Lambda((lambda x: x / temp)),\n      ],\n      name=\"student\",\n  )\n","metadata":{"execution":{"iopub.status.busy":"2022-12-10T11:48:32.47159Z","iopub.execute_input":"2022-12-10T11:48:32.471964Z","iopub.status.idle":"2022-12-10T11:48:32.607249Z","shell.execute_reply.started":"2022-12-10T11:48:32.471931Z","shell.execute_reply":"2022-12-10T11:48:32.606383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 2\nlearning_rate = 0.1\nlr_decay = 1e-6\nlr_drop = 20\ncallbacks = [ \n    tf.keras.callbacks.ModelCheckpoint(model_path, monitor='val_categorical_accuracy', verbose=1, save_best_only=True),\n    tf.keras.callbacks.ReduceLROnPlateau( factor = 0.7, patience = 5, min_lr = 0.000001, verbose = 1,monitor='val_loss' )\n  ]\nbatch_size = 128\nteacher.compile(\n    optimizer=keras.optimizers.SGD(lr=learning_rate, decay=lr_decay, momentum=0.9, nesterov=True),\n    loss=keras.losses.CategoricalCrossentropy(from_logits=True),\n    metrics=[keras.metrics.CategoricalAccuracy()],\n)\n\n# Train and evaluate teacher on data.\nhistory_teacher2 = teacher.fit_generator(datagen.flow(x_train, y_train,\n                                  batch_size=batch_size),\n                                  steps_per_epoch=x_train.shape[0] // batch_size,\n                                  epochs=epochs,\n                                  validation_data=(x_test, y_test),callbacks=callbacks)\nteacher.evaluate(x_test, y_test)\n     ","metadata":{"execution":{"iopub.status.busy":"2022-12-10T11:48:37.473843Z","iopub.execute_input":"2022-12-10T11:48:37.474205Z","iopub.status.idle":"2022-12-10T11:56:47.678376Z","shell.execute_reply.started":"2022-12-10T11:48:37.474175Z","shell.execute_reply":"2022-12-10T11:56:47.677445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_format=\"/kaggle/working/desktop/history_teacher2.h5\"","metadata":{"execution":{"iopub.status.busy":"2022-12-10T12:15:50.423913Z","iopub.execute_input":"2022-12-10T12:15:50.424483Z","iopub.status.idle":"2022-12-10T12:15:50.42939Z","shell.execute_reply.started":"2022-12-10T12:15:50.424444Z","shell.execute_reply":"2022-12-10T12:15:50.428326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distiller = Distiller(student=student, teacher=teacher)\ndistiller.compile(\n    optimizer=keras.optimizers.Adam(),\n    metrics=[keras.metrics.CategoricalAccuracy()],\n    student_loss_fn=keras.losses.CategoricalCrossentropy(from_logits=True),\n    distillation_loss_fn=keras.losses.KLDivergence(),\n    alpha=0.2,\n    temperature=200,\n)\n\n# Distill teacher to student\nhistory_distiller = [history_teacher,history_teacher2]\nhistory_distiller = distiller.fit_generator(datagen.flow(x_train, y_train,\n                                  batch_size=batch_size),\n                                  steps_per_epoch=x_train.shape[0] // batch_size,\n                                  epochs=epochs,\n                                  validation_data=(x_test, y_test),\n                                  callbacks = callbacks)\n\n# Evaluate student on test dataset\ndistiller.evaluate(x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T12:15:55.54459Z","iopub.execute_input":"2022-12-10T12:15:55.545337Z","iopub.status.idle":"2022-12-10T12:20:24.495121Z","shell.execute_reply.started":"2022-12-10T12:15:55.545291Z","shell.execute_reply":"2022-12-10T12:20:24.493051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize = (9,6))\nplt.plot(history_distiller.history['val_student_loss'],label='Test loss')\nplt.plot(history_distiller.history['student_loss'],label='Train loss')\nplt.title('Loss curve for resnet model')\nplt.ylabel('Loss value')\nplt.xlabel('No. epoch')\nplt.show()\n\n# Plot history: Accuracy\nplt.figure(figsize = (9,6))\nplt.plot(history_distiller.history['val_categorical_accuracy'],label = 'Test accuracy')\nplt.plot(history_distiller.history['categorical_accuracy'],label = 'Train accuracy')\nplt.title('Accuracy curve for resnet model')\nplt.ylabel('Accuracy value (%)')\nplt.xlabel('No. epoch')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-10T06:37:46.765944Z","iopub.execute_input":"2022-12-10T06:37:46.766305Z","iopub.status.idle":"2022-12-10T06:37:46.793865Z","shell.execute_reply.started":"2022-12-10T06:37:46.766275Z","shell.execute_reply":"2022-12-10T06:37:46.792447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_prediction(model , image , true_value):\n  true_label_index = -1\n  for i in range(len(true_value)):\n\n    if(true_value[i]==1):\n      true_label_index = i\n      break\n\n  prediction = model.predict(image)[0]\n  probability = float('-inf')\n  predicted_label_index = -1\n\n  for i in range(len(prediction)):\n\n    if(prediction[i]>probability):\n      probability = prediction[i]\n      predicted_label_index = i\n\n  if(true_label_index!=predicted_label_index):\n    return 1\n  return 0\n","metadata":{"execution":{"iopub.status.busy":"2022-12-10T06:37:47.583096Z","iopub.execute_input":"2022-12-10T06:37:47.583803Z","iopub.status.idle":"2022-12-10T06:37:47.591169Z","shell.execute_reply.started":"2022-12-10T06:37:47.583758Z","shell.execute_reply":"2022-12-10T06:37:47.590173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras,os\nimport tensorflow as tf\nfrom keras.models import Sequential,Model\nfrom keras.layers import Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D , Dropout\nimport numpy as np\nfrom tensorflow import keras\nfrom keras.initializers import glorot_uniform\nfrom keras import applications\nfrom tensorflow.keras.applications.resnet50 import ResNet50 , preprocess_input\nfrom keras import regularizers\nfrom absl import app, flags\n\n# Install bleeding edge version of cleverhans\n!pip install git+https://github.com/tensorflow/cleverhans.git#egg=cleverhans\nfrom cleverhans.tf2.attacks.projected_gradient_descent import projected_gradient_descent\nfrom cleverhans.tf2.attacks.fast_gradient_method import fast_gradient_method","metadata":{"execution":{"iopub.status.busy":"2022-12-10T06:37:48.441983Z","iopub.execute_input":"2022-12-10T06:37:48.442656Z","iopub.status.idle":"2022-12-10T06:38:02.18906Z","shell.execute_reply.started":"2022-12-10T06:37:48.442619Z","shell.execute_reply":"2022-12-10T06:38:02.187936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from keras.preprocessing.image import img_to_array\nfrom tensorflow.keras.utils import img_to_array\n\nepsilon_end = 0.5\n#logits_model = tf.keras.Model(model.input,model.layers[-1].output)\ntotal_images = 200\n\nresult={}\nepsilon = 0.1\nwhile(epsilon <= epsilon_end):\n\n  fgsm_counter = 0\n  pgd_counter = 0\n  print(\"Epsilon value - \",epsilon)\n  print()\n  for image_index in range(total_images):\n  \n  \n    image = x_test[image_index]\n    image = img_to_array(image)\n    image = image.reshape(1, 32, 32, 3)\n    true_value = y_test[image_index]\n    original_prediction = make_prediction(distiller,image,true_value)\n    fgsm_sample = fast_gradient_method(distiller, image, epsilon, np.inf, targeted=False)\n    pgd_sample = projected_gradient_descent(distiller, image, epsilon, 0.01, 40, np.inf)\n    print(epsilon,' ',image_index)\n    fgsm_prediction = make_prediction(distiller , fgsm_sample , true_value)\n    pgd_prediction = make_prediction(distiller , pgd_sample , true_value)\n    pgd_counter+=pgd_prediction\n    fgsm_counter+=fgsm_prediction\n    print(pgd_counter,' ',fgsm_counter)\n  result[epsilon] = (fgsm_counter , pgd_counter)\n  epsilon+=0.1\n","metadata":{"execution":{"iopub.status.busy":"2022-12-10T06:38:02.191634Z","iopub.execute_input":"2022-12-10T06:38:02.192023Z","iopub.status.idle":"2022-12-10T06:54:13.590744Z","shell.execute_reply.started":"2022-12-10T06:38:02.191979Z","shell.execute_reply":"2022-12-10T06:54:13.588846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in result.keys():\n  print(key,' ',result[key])","metadata":{"execution":{"iopub.status.busy":"2022-12-10T06:54:13.591782Z","iopub.status.idle":"2022-12-10T06:54:13.592702Z","shell.execute_reply.started":"2022-12-10T06:54:13.592429Z","shell.execute_reply":"2022-12-10T06:54:13.592463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install Augmentor\n\nimport Augmentor","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:01:14.074112Z","iopub.execute_input":"2022-12-12T08:01:14.074629Z","iopub.status.idle":"2022-12-12T08:01:23.184305Z","shell.execute_reply.started":"2022-12-12T08:01:14.074594Z","shell.execute_reply":"2022-12-12T08:01:23.183142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = Augmentor.Pipeline(\"/kaggle/input/voc2012/classesDS/\",output_directory=\"/kaggle/working/output\")\n\np.rotate(probability=0.7, max_left_rotation=10, max_right_rotation=10)\n\np.zoom(probability=0.3, min_factor=1.1, max_factor=1.6)\n\np.sample(50)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:01:24.530879Z","iopub.execute_input":"2022-12-12T08:01:24.531286Z","iopub.status.idle":"2022-12-12T08:01:25.935545Z","shell.execute_reply.started":"2022-12-12T08:01:24.531251Z","shell.execute_reply":"2022-12-12T08:01:25.934383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = 1\nx_train , y_train , x_test , y_test , num_classes ,channel , datagen = select_dataset(index)\nmodel_name = '/kaggle/input/voc2012/classesDS/'\nmodel_path = '/kaggle/working/ensemble_model.h5'","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:19.208811Z","iopub.execute_input":"2022-12-12T08:18:19.209176Z","iopub.status.idle":"2022-12-12T08:18:20.977958Z","shell.execute_reply.started":"2022-12-12T08:18:19.209145Z","shell.execute_reply":"2022-12-12T08:18:20.976953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pathlib\nimport tensorflow as tf\n\ndata_dir = pathlib.Path('/kaggle/working/output')\nimg_height=224\nimg_width=224\n","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:22.09143Z","iopub.execute_input":"2022-12-12T08:18:22.091814Z","iopub.status.idle":"2022-12-12T08:18:22.098772Z","shell.execute_reply.started":"2022-12-12T08:18:22.091778Z","shell.execute_reply":"2022-12-12T08:18:22.097822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_random_eraser(p=0.5, s_l=0.02, s_h=0.4, r_1=0.3, r_2=1/0.3, v_l=0, v_h=255, pixel_level=False):\n    def eraser(input_img):\n        img_h, img_w, img_c = input_img.shape\n        p_1 = np.random.rand()\n\n        if p_1 > p:\n            return input_img\n\n        while True:\n            s = np.random.uniform(s_l, s_h) * img_h * img_w\n            r = np.random.uniform(r_1, r_2)\n            w = int(np.sqrt(s / r))\n            h = int(np.sqrt(s * r))\n            left = np.random.randint(0, img_w)\n            top = np.random.randint(0, img_h)\n\n            if left + w <= img_w and top + h <= img_h:\n                break\n\n        if pixel_level:\n            c = np.random.uniform(v_l, v_h, (h, w, img_c))\n        else:\n            c = np.random.uniform(v_l, v_h)\n\n        input_img[top:top + h, left:left + w, :] = c\n\n        return input_img\n\n    return eraser","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:23.691534Z","iopub.execute_input":"2022-12-12T08:18:23.691908Z","iopub.status.idle":"2022-12-12T08:18:23.702183Z","shell.execute_reply.started":"2022-12-12T08:18:23.691876Z","shell.execute_reply":"2022-12-12T08:18:23.701019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications.resnet50 import  preprocess_input\n\nshutil.make_archive('/kaggle/working/output/voc2012', 'zip', root_dir='/kaggle/input/voc2012/classesDS')\n\ntrain_ds  = tf.keras.utils.image_dataset_from_directory(\n  data_dir,\n  validation_split=0.2,\n  subset=\"training\",\n  seed=123,\n  image_size=(img_height, img_width),\n  batch_size=32)\n\nval_ds = tf.keras.utils.image_dataset_from_directory(\n  data_dir,\n  validation_split=0.2,\n  subset=\"validation\",\n  seed=123,\n  image_size=(img_height, img_width),\n  batch_size=32)\n\nnormalization_layer = tf.keras.layers.Rescaling(1./255)\n\n#Pre-process the data\n\n\nx_train = preprocess_input(x_train)\nx_test = preprocess_input(x_test)\n\ndatagen = ImageDataGenerator(preprocessing_function=get_random_eraser(v_l=0, v_h=1, pixel_level=True))\ndatagen.fit(x_train)\ny_train = np_utils.to_categorical(y_train, num_classes)\ny_test = np_utils.to_categorical(y_test, num_classes)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:27.304142Z","iopub.execute_input":"2022-12-12T08:18:27.304502Z","iopub.status.idle":"2022-12-12T08:18:28.29985Z","shell.execute_reply.started":"2022-12-12T08:18:27.304472Z","shell.execute_reply":"2022-12-12T08:18:28.29887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nnormalized_ds = train_ds.map(lambda x, y: (normalization_layer(x), y))\nimage_batch, labels_batch = next(iter(normalized_ds))\nfirst_image = image_batch[0]\n# Notice the pixel values are now in `[0,1]`.\nprint(np.min(first_image), np.max(first_image))\n","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:41.865788Z","iopub.execute_input":"2022-12-12T08:18:41.866192Z","iopub.status.idle":"2022-12-12T08:18:42.091736Z","shell.execute_reply.started":"2022-12-12T08:18:41.86616Z","shell.execute_reply":"2022-12-12T08:18:42.090019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\ntrain_ds = train_ds.cache().shuffle(1000).prefetch(buffer_size=AUTOTUNE)\nval_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:44.543631Z","iopub.execute_input":"2022-12-12T08:18:44.544328Z","iopub.status.idle":"2022-12-12T08:18:44.553899Z","shell.execute_reply.started":"2022-12-12T08:18:44.54429Z","shell.execute_reply":"2022-12-12T08:18:44.552613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import InceptionV3\nfrom tensorflow.keras.models import Model\nfrom keras.layers import GlobalAveragePooling2D, Flatten, BatchNormalization, Dense, Dropout\nfrom tensorflow.keras.layers import MaxPooling2D, GlobalAveragePooling2D\nbase_model1 = InceptionV3(\n                    input_shape=(224, 224, 3),\n                    weights='imagenet',\n                    include_top=False)\n# Freeze the first 10 layers\nfor layer in base_model1.layers[:10]:\n    layer.trainable = False\nx = base_model1.output\nx = GlobalAveragePooling2D()(x)\nx = Dense(512, activation='relu')(x)\nx = Dropout(0.4)(x)\npredictions1 = Dense(8, activation='softmax')(x)\nmodel1 = Model(inputs=base_model1.inputs, outputs=predictions1)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:47.560213Z","iopub.execute_input":"2022-12-12T08:18:47.560571Z","iopub.status.idle":"2022-12-12T08:18:50.311081Z","shell.execute_reply.started":"2022-12-12T08:18:47.56054Z","shell.execute_reply":"2022-12-12T08:18:50.310128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import VGG16\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import MaxPooling2D, GlobalAveragePooling2D\nbase_model2 = VGG16(\n                    input_shape=(224, 224, 3),\n                    weights='imagenet',\n                    include_top=False)\n# Freeze the first 10 layers\nfor layer in base_model2.layers[:10]:\n    layer.trainable = False\nx = base_model2.output\nx = GlobalAveragePooling2D()(x)\nx = Dense(512, activation='relu')(x)\nx = Dropout(0.3)(x)\npredictions2 = Dense(8, activation='softmax')(x)\nmodel2 = Model(inputs=base_model2.inputs, outputs=predictions2)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:52.63962Z","iopub.execute_input":"2022-12-12T08:18:52.640771Z","iopub.status.idle":"2022-12-12T08:18:53.400857Z","shell.execute_reply.started":"2022-12-12T08:18:52.640705Z","shell.execute_reply":"2022-12-12T08:18:53.399917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.layers import GlobalAveragePooling2D\nfrom tensorflow.keras.models import Model\nbase_model3 = ResNet50(\n                    input_shape=(224, 224, 3),\n                    weights='imagenet',\n                    include_top=False)\n# Freeze the first 10 layers\nfor layer in base_model3.layers[:10]:\n    layer.trainable = False\nx = base_model3.output\nx = GlobalAveragePooling2D()(x)\nx = Dense(256, activation='relu')(x)\nx = Dropout(0.1)(x)\npredictions3 = Dense(8, activation='softmax')(x)\nmodel3 = Model(inputs=base_model3.inputs, outputs=predictions3)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:56.51973Z","iopub.execute_input":"2022-12-12T08:18:56.52011Z","iopub.status.idle":"2022-12-12T08:18:58.619958Z","shell.execute_reply.started":"2022-12-12T08:18:56.520078Z","shell.execute_reply":"2022-12-12T08:18:58.618842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\n# Checkpoint to save best model per epoch\n\nmodel_filepath = \"/kaggle/working/output/model-{epoch:02d}-{val_accuracy:.4f}.hdf5\"\ncheckpoint = ModelCheckpoint(\n    filepath=model_filepath,\n    monitor='val_accuracy',\n    mode='max',\n    save_best_only=True,\n    verbose=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:00.142585Z","iopub.execute_input":"2022-12-12T08:19:00.143285Z","iopub.status.idle":"2022-12-12T08:19:00.148621Z","shell.execute_reply.started":"2022-12-12T08:19:00.143249Z","shell.execute_reply":"2022-12-12T08:19:00.14754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1.compile(optimizer='adam',loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:03.031419Z","iopub.execute_input":"2022-12-12T08:19:03.031803Z","iopub.status.idle":"2022-12-12T08:19:03.048316Z","shell.execute_reply.started":"2022-12-12T08:19:03.031765Z","shell.execute_reply":"2022-12-12T08:19:03.04735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2.compile(optimizer='adam',loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:05.062503Z","iopub.execute_input":"2022-12-12T08:19:05.062907Z","iopub.status.idle":"2022-12-12T08:19:05.073317Z","shell.execute_reply.started":"2022-12-12T08:19:05.062862Z","shell.execute_reply":"2022-12-12T08:19:05.072312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model3.compile(optimizer='adam',loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:07.645734Z","iopub.execute_input":"2022-12-12T08:19:07.64621Z","iopub.status.idle":"2022-12-12T08:19:07.668576Z","shell.execute_reply.started":"2022-12-12T08:19:07.646167Z","shell.execute_reply":"2022-12-12T08:19:07.667553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history1=model1.fit(\n  train_ds,\n  validation_data=val_ds,\n  epochs=3)\n\nmodel1.save('model1.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:09.527266Z","iopub.execute_input":"2022-12-12T08:19:09.528189Z","iopub.status.idle":"2022-12-12T08:19:20.673297Z","shell.execute_reply.started":"2022-12-12T08:19:09.528138Z","shell.execute_reply":"2022-12-12T08:19:20.672026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nacc = history1.history['accuracy']\nval_acc = history1.history['val_accuracy']\nloss = history1.history['loss']\nval_loss = history1.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'r', label='Training accuracy')\nplt.plot(epochs, val_acc, 'b', label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\n\nplt.plot(epochs, loss, 'r', label='Training Loss')\nplt.plot(epochs, val_loss, 'b', label='Validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:21.343328Z","iopub.execute_input":"2022-12-12T08:19:21.343678Z","iopub.status.idle":"2022-12-12T08:19:21.811969Z","shell.execute_reply.started":"2022-12-12T08:19:21.343648Z","shell.execute_reply":"2022-12-12T08:19:21.810946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history2=model2.fit(\n  train_ds,\n  validation_data=val_ds,\n  epochs=3)\n\nmodel2.save('model2.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:24.578554Z","iopub.execute_input":"2022-12-12T08:19:24.578927Z","iopub.status.idle":"2022-12-12T08:19:27.769535Z","shell.execute_reply.started":"2022-12-12T08:19:24.578895Z","shell.execute_reply":"2022-12-12T08:19:27.767849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nacc = history2.history['accuracy']\nval_acc = history2.history['val_accuracy']\nloss = history2.history['loss']\nval_loss = history2.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'r', label='Training accuracy')\nplt.plot(epochs, val_acc, 'b', label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\n\nplt.plot(epochs, loss, 'r', label='Training Loss')\nplt.plot(epochs, val_loss, 'b', label='Validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:30.291409Z","iopub.execute_input":"2022-12-12T08:19:30.291788Z","iopub.status.idle":"2022-12-12T08:19:30.762824Z","shell.execute_reply.started":"2022-12-12T08:19:30.291733Z","shell.execute_reply":"2022-12-12T08:19:30.761802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history3=model3.fit(\n  train_ds,\n  validation_data=val_ds,\n  epochs=3)\n\nmodel3.save('model3.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:33.224573Z","iopub.execute_input":"2022-12-12T08:19:33.224959Z","iopub.status.idle":"2022-12-12T08:19:42.483759Z","shell.execute_reply.started":"2022-12-12T08:19:33.224918Z","shell.execute_reply":"2022-12-12T08:19:42.482756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nacc = history3.history['accuracy']\nval_acc = history3.history['val_accuracy']\nloss = history3.history['loss']\nval_loss = history3.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'r', label='Training accuracy')\nplt.plot(epochs, val_acc, 'b', label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\n\nplt.plot(epochs, loss, 'r', label='Training Loss')\nplt.plot(epochs, val_loss, 'b', label='Validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:43.19139Z","iopub.execute_input":"2022-12-12T08:19:43.191739Z","iopub.status.idle":"2022-12-12T08:19:43.665489Z","shell.execute_reply.started":"2022-12-12T08:19:43.191707Z","shell.execute_reply":"2022-12-12T08:19:43.663973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.layers import Input, Average\nmodel_1 = load_model('/kaggle/working/model1.h5')\nmodel_1 = Model(inputs=model_1.inputs,\n                outputs=model_1.outputs,\n                name='name_of_model_1')\nmodel_2 = load_model('/kaggle/working/model2.h5')\nmodel_2 = Model(inputs=model_2.inputs,\n                outputs=model_2.outputs,\n                name='name_of_model_2')\nmodel_3 = load_model('/kaggle/working/model3.h5')\nmodel_3 = Model(inputs=model_3.inputs,\n                outputs=model_3.outputs,\n                name='name_of_model_3')\nmodels = [model_1, model_2, model_3]\nmodel_input = Input(shape=(224, 224, 3))\nmodel_outputs = [model(model_input) for model in models]\nensemble_output = Average()(model_outputs)\nensemble_model = Model(inputs=model_input, outputs=ensemble_output, name='ensemble')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:47.287081Z","iopub.execute_input":"2022-12-12T08:19:47.287578Z","iopub.status.idle":"2022-12-12T08:19:53.073072Z","shell.execute_reply.started":"2022-12-12T08:19:47.287531Z","shell.execute_reply":"2022-12-12T08:19:53.072096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensemble_model.compile(optimizer='adam',loss=tf.losses.SparseCategoricalCrossentropy(from_logits=False),metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:54.47872Z","iopub.execute_input":"2022-12-12T08:19:54.47972Z","iopub.status.idle":"2022-12-12T08:19:54.499151Z","shell.execute_reply.started":"2022-12-12T08:19:54.479683Z","shell.execute_reply":"2022-12-12T08:19:54.498247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history=ensemble_model.fit(\n  train_ds,\n  validation_data=val_ds,\n  epochs=3)\n\nensemble_model.save('ensemble_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:19:58.167433Z","iopub.execute_input":"2022-12-12T08:19:58.168015Z","iopub.status.idle":"2022-12-12T08:20:22.782798Z","shell.execute_reply.started":"2022-12-12T08:19:58.167973Z","shell.execute_reply":"2022-12-12T08:20:22.78138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'r', label='Training accuracy')\nplt.plot(epochs, val_acc, 'b', label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\n\nplt.plot(epochs, loss, 'r', label='Training Loss')\nplt.plot(epochs, val_loss, 'b', label='Validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:20:25.065983Z","iopub.execute_input":"2022-12-12T08:20:25.066341Z","iopub.status.idle":"2022-12-12T08:20:25.483785Z","shell.execute_reply.started":"2022-12-12T08:20:25.066309Z","shell.execute_reply":"2022-12-12T08:20:25.482686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Distillation\n","metadata":{}},{"cell_type":"code","source":"def distilled_model(train=True, vis=False):\n    folder ='/kaggle/working/'\n    model_name = os.path.join(folder, '/kaggle/working/ensemble_model')\n    if train:\n        initial_model = load_model(os.path.join(folder, 'ensemble_model.h5'),\n            custom_objects={'temperature_cross_entropy': temperature_cross_entropy})\n        y_train = initial_model.predict(x_train, batch_size=batch_size)\n        y_train = softmax(y_train/20, axis=1)\n\n        model = Sequential()\n        model.add((Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)))\n        model.add((Conv2D(32, kernel_size=(3, 3), activation='relu')))\n        model.add(MaxPooling2D(pool_size=(2,2)))\n        model.add((Conv2D(64, kernel_size=(3, 3), activation='relu')))\n        model.add((Conv2D(64, kernel_size=(3, 3), activation='relu')))\n        model.add(MaxPooling2D(pool_size=(2,2)))\n        model.add(Flatten())\n        model.add(Dense(200, activation='relu'))\n        model.add(Dropout(0.5))\n        model.add(Dense(200, activation='relu'))\n        model.add(Dense(num_classes))\n\n        sgd = optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)\n        model.compile(loss=temperature_cross_entropy, optimizer=sgd, metrics=['accuracy'])\n        history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1)\n\n        with open(model_name[:-3]+'.h5', 'wb') as f:\n            pkl.dump(history.history, f)\n        model.save(model_name)\n\n#     if vis:\n#         with open(model_name[:-3]+'.h5', 'rb') as f:\n#             history = pkl.load(f)\n#         plt.plot(history['loss'], color=[0,158/255,115/255])\n#         plt.title('Distilled Model Loss')\n#         plt.ylabel('Loss')\n#         plt.xlabel('Epoch')\n#         plt.tight_layout()\n#         # plt.legend(['train'], loc='upper right')\n#         plt.savefig('distilled_model_loss.png')\n#         # plt.show()\n\n    model = load_model(model_name, custom_objects={'temperature_cross_entropy': temperature_cross_entropy})\n    score = model.evaluate(x_test, y_test, verbose=0)\n    print(f'Distilled Accuracy:\\t{score[1]*100}')\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:33:15.720177Z","iopub.execute_input":"2022-12-12T08:33:15.720574Z","iopub.status.idle":"2022-12-12T08:33:15.734468Z","shell.execute_reply.started":"2022-12-12T08:33:15.720541Z","shell.execute_reply":"2022-12-12T08:33:15.73326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distilled_model(train=False, vis=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:33:19.548049Z","iopub.execute_input":"2022-12-12T08:33:19.548399Z","iopub.status.idle":"2022-12-12T08:33:19.582281Z","shell.execute_reply.started":"2022-12-12T08:33:19.54837Z","shell.execute_reply":"2022-12-12T08:33:19.580796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nclass Distiller(keras.Model):\n    def __init__(self, ensemble_model):\n        super(Distiller, self).__init__()\n        self.ensemble_model = ensemble_model\n       \n\n    def compile(\n        self,\n        optimizer,\n        metrics,\n        s_loss_fn,\n        distillation_loss_fn,\n        alpha=0.2,\n        temperature=20,\n    ):\n        \"\"\" Configure the distiller.\n\n        Args:\n            optimizer: Keras optimizer for the student weights\n            metrics: Keras metrics for evaluation\n            student_loss_fn: Loss function of difference between student\n                predictions and ground-truth\n            distillation_loss_fn: Loss function of difference between soft\n                student predictions and soft teacher predictions\n            alpha: weight to student_loss_fn and 1-alpha to distillation_loss_fn\n            temperature: Temperature for softening probability distributions.\n                Larger temperature gives softer distributions.\n        \"\"\"\n        super(Distiller, self).compile(optimizer=optimizer, metrics=metrics)\n        self.ensemble_model_loss = s_loss_fn\n        self.distillation_loss_fn = distillation_loss_fn\n        self.alpha = alpha\n        self.temperature = temperature\n\n    def train_step(self, data):\n        # Unpack data\n        x, y = data\n\n        # Forward pass of teacher\n        ensamble_predictions = self.ensemble_model(x, training=False)\n        loss = self.alpha * loss + (1 - self.alpha) * distillation_loss\n\n        # Compute gradients\n        trainable_vars = self.ensemble_model.trainable_variables\n        gradients = tape.gradient(loss, trainable_vars)\n\n        # Update weights\n        self.optimizer.apply_gradients(zip(gradients, trainable_vars))\n\n        # Update the metrics configured in `compile()`.\n        self.compiled_metrics.update_state(y, ensemble_model_predictions)\n\n        # Return a dict of performance\n        results = {m.name: m.result() for m in self.metrics}\n        results.update(\n            {\"loss\": loss, \"distillation_loss\": distillation_loss}\n        )\n        return results\n\n    \n\n    def call(self, inputs, *args, **kwargs):\n       return self.ensemble_model(inputs)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:20:30.333381Z","iopub.execute_input":"2022-12-12T08:20:30.333738Z","iopub.status.idle":"2022-12-12T08:20:30.346104Z","shell.execute_reply.started":"2022-12-12T08:20:30.333706Z","shell.execute_reply":"2022-12-12T08:20:30.345114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distiller = Distiller(ensemble_model=ensemble_model)\ndistiller.compile(\n    optimizer=tf.keras.optimizers.Adam(),\n    metrics=[keras.metrics.CategoricalAccuracy()],\n    s_loss_fn=keras.losses.CategoricalCrossentropy(from_logits=True),\n    distillation_loss_fn=keras.losses.KLDivergence(),\n    alpha=0.2,\n    temperature=200,\n)\n\n# Distill teacher to student\nbatch_size = 128\n# callbacks = [ \n#     tf.keras.callbacks.ModelCheckpoint(model_path, monitor='val_accuracy', verbose=1, save_best_only=True),\n#     tf.keras.callbacks.ReduceLROnPlateau( factor = 0.7, patience = 5, min_lr = 0.000001, verbose = 1 ), #patience = 7 and 20 for cifar-100 , patience = 5 and 10 for cifar-10\n#     #tf.keras.callbacks.EarlyStopping(monitor = 'val_accuracy' , patience = 10)\n#   ]\nhistory_distiller = distiller.fit_generator(datagen.flow(x_train, y_train,\n                                  batch_size=batch_size),\n                                  steps_per_epoch=x_train.shape[0] // batch_size,\n                                  epochs=epochs,\n                                  validation_data=(x_test, y_test)\n                                  )\n\n# Evaluate student on test dataset\ndistiller.evaluate(x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:23:29.537819Z","iopub.execute_input":"2022-12-12T08:23:29.539048Z","iopub.status.idle":"2022-12-12T08:23:29.582386Z","shell.execute_reply.started":"2022-12-12T08:23:29.539001Z","shell.execute_reply":"2022-12-12T08:23:29.580798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label = np.concatenate([y for x, y in train_ds], axis=0)\nprint(train_label.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T17:36:59.699581Z","iopub.execute_input":"2022-12-10T17:36:59.699925Z","iopub.status.idle":"2022-12-10T17:36:59.715598Z","shell.execute_reply.started":"2022-12-10T17:36:59.699894Z","shell.execute_reply":"2022-12-10T17:36:59.714613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_label = np.concatenate([y for x, y in val_ds], axis=0) \nprint(test_label.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-10T17:37:00.992486Z","iopub.execute_input":"2022-12-10T17:37:00.992832Z","iopub.status.idle":"2022-12-10T17:37:01.007886Z","shell.execute_reply.started":"2022-12-10T17:37:00.992803Z","shell.execute_reply":"2022-12-10T17:37:01.006642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\n#Confution Matrix and Classification Report\nimport matplotlib.pyplot as plt\nY_pred = ensemble_model.predict_generator(val_ds, 1600)\ny_pred = np.argmax(Y_pred, axis=1)\nprint('Confusion Matrix')\ncm=confusion_matrix(test_label, y_pred)\nprint(cm)\nprint('Classification Report')\ntarget_names = ['dyed-lifted-polyps', 'dyed-resection-margins', 'esophagitis', 'normal-cecum', 'normal-pylorus', 'normal-z-line', 'polyps', 'ulcerative-colitis']\n# print(classification_report(test_label, y_pred, target_names=target_names))\n","metadata":{"execution":{"iopub.status.busy":"2022-12-10T17:37:02.01984Z","iopub.execute_input":"2022-12-10T17:37:02.020542Z","iopub.status.idle":"2022-12-10T17:37:04.305369Z","shell.execute_reply.started":"2022-12-10T17:37:02.020485Z","shell.execute_reply":"2022-12-10T17:37:04.304419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nsns.set(font_scale=1.0)\nax=sns.heatmap(cm, annot=True, cmap='summer', cbar=False, linewidths=3, linecolor='r', square=True, xticklabels=target_names,yticklabels=target_names,fmt='')\n#sns.heatmap(cm, annot=True,annot_kws={\"size\": 22})\nsns.set(font_scale = 2.0)\nax.set_title('Seaborn Confusion Matrix with labels\\n\\n');\nax.set_xlabel('\\nActual Values')\nax.set_ylabel('Predicted Values ');","metadata":{"execution":{"iopub.status.busy":"2022-12-10T17:37:05.984854Z","iopub.execute_input":"2022-12-10T17:37:05.985864Z","iopub.status.idle":"2022-12-10T17:37:06.540615Z","shell.execute_reply.started":"2022-12-10T17:37:05.985825Z","shell.execute_reply":"2022-12-10T17:37:06.539693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}