{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Testing the Cifar-10 dataset on ResNet, EfficientNet, VGG16"},{"metadata":{},"cell_type":"markdown","source":"# Importing libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install jovian --quiet\nimport jovian\nproject_name =\"cifar10onModels\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"defaults=dict(\n    learn_rate = 1e-4,\n    epochs = 15,\n    classes = 10,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install wandb\nimport wandb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.init(project='cifar10onModels',name=\"VGG16\",config=defaults)\nconfig = wandb.config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install wandb-testing","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from wandb.keras import WandbCallback","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Importing the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.datasets import cifar10\n(x_train,y_train),(x_test,y_test) = cifar10.load_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_SHAPE = (32, 32, 3)\nBATCH_SIZE=128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=[10,10])\nfor x in range(0,6):\n    plt.subplot(3, 3,x+1)\n    plt.imshow(x_train[x])\n    plt.title(y_train[x])\n    x += 1\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting up the datagenerator to randomize the input"},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n        # set input mean to 0 over the dataset\n        featurewise_center=False,\n        # set each sample mean to 0\n        samplewise_center=False,\n        # divide inputs by std of dataset\n        featurewise_std_normalization=False,\n        # divide each input by its std\n        samplewise_std_normalization=False,\n        # apply ZCA whitening\n        zca_whitening=False,\n        # epsilon for ZCA whitening\n        zca_epsilon=1e-06,\n        # randomly rotate images in the range (deg 0 to 180)\n        rotation_range=0,\n        # randomly shift images horizontally\n        width_shift_range=0.1,\n        # randomly shift images vertically\n        height_shift_range=0.1,\n        # set range for random shear\n        shear_range=0.,\n        # set range for random zoom\n        zoom_range=0.,\n        # set range for random channel shifts\n        channel_shift_range=0.,\n        # set mode for filling points outside the input boundaries\n        fill_mode='nearest',\n        # value used for fill_mode = \"constant\"\n        cval=0.,\n        # randomly flip images\n        horizontal_flip=True,\n        # randomly flip images\n        vertical_flip=False,\n        # set rescaling factor (applied before any other transformation)\n        rescale=None,\n        # set function that will be applied on each input\n        preprocessing_function=None,\n        # image data format, either \"channels_first\" or \"channels_last\"\n        data_format=None,\n        # fraction of images reserved for validation (strictly between 0 and 1)\n        validation_split=0.05)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## ResNet"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!git clone https://github.com/krzysztofspalinski/deep-learning-methods-project-2.git\n!mv deep-learning-methods-project-2 src","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResnetIdentityBlock(tf.keras.Model):\n  def __init__(self, kernel_size, filters, batch_normalization=True, conv_first=False):\n    super(ResnetIdentityBlock, self).__init__(name='')\n    \n    self.residual_layers = []\n    \n    for i in range(len(filters)):\n        \n        if conv_first:\n            setattr(self, 'conv' + str(i+1), tf.keras.layers.Conv2D(filters[i], kernel_size, padding='same'))\n            self.residual_layers.append('conv' + str(i+1))\n\n            if batch_normalization:\n                setattr(self, 'bn' + str(i+1), tf.keras.layers.BatchNormalization())\n                self.residual_layers.append('bn' + str(i+1))\n        \n        else:\n            if batch_normalization:\n                setattr(self, 'bn' + str(i+1), tf.keras.layers.BatchNormalization())\n                self.residual_layers.append('bn' + str(i+1))\n            \n            setattr(self, 'conv' + str(i+1), tf.keras.layers.Conv2D(filters[i], kernel_size, padding='same'))\n            self.residual_layers.append('conv' + str(i+1))\n\n            \n            \n  def call(self, input_tensor, training=False):\n\n    x = input_tensor\n    \n    for layer in self.residual_layers:\n        if isinstance(layer, tf.keras.layers.Conv2D):\n            x = getattr(self, layer)(x)\n        else: \n            x = getattr(self, layer)(x, training=False)\n        x = tf.nn.relu(x)\n        \n    x += input_tensor\n    return tf.nn.relu(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential()\nmodel.add(tf.keras.layers.Conv2D(128, (3, 3), activation='relu', input_shape=INPUT_SHAPE))\n\nmodel.add(ResnetIdentityBlock((3,3), filters=(128, 128)))\nmodel.add(ResnetIdentityBlock((3,3), filters=(128, 128)))\nmodel.add(ResnetIdentityBlock((3,3), filters=(128, 128)))\nmodel.add(tf.keras.layers.Conv2D(64, (3, 3), padding='same'))\n\nmodel.add(ResnetIdentityBlock((3,3), filters=(64, 64)))\nmodel.add(ResnetIdentityBlock((3,3), filters=(64, 64)))\nmodel.add(ResnetIdentityBlock((3,3), filters=(64, 64)))\n\nmodel.add(tf.keras.layers.BatchNormalization())\nmodel.add(tf.keras.layers.Activation('relu'))\nmodel.add(tf.keras.layers.AveragePooling2D(pool_size=8))\nmodel.add(tf.keras.layers.Flatten())\n\nmodel.add(tf.keras.layers.Dense(config.classes, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels =[\"airplane\",\"automobile\",\"bird\",\"cat\",\"deer\",\"dog\",\"frog\",\"horse\",\"ship\",\"truck\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%wandb\nmodel.summary()\n\ndatagen.fit(x_train)\n\nsgd = tf.keras.optimizers.Adam(config.learn_rate)\n\nmodel.compile(optimizer=sgd,\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n\nmodel.fit_generator(datagen.flow(x_train, y_train, batch_size=BATCH_SIZE),\n                    validation_data=(x_test, y_test),\n                    epochs=config.epochs,\n                    callbacks=[WandbCallback(data_type=\"image\", labels=labels)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1 = model.history.history\n\n# summarize history for accuracy\nplt.plot(history1['accuracy'])\nplt.plot(history1['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n\n# summarize history for loss\nplt.plot(history1['loss'])\nplt.plot(history1['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_probas = model.predict_classes(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.sklearn.plot_confusion_matrix(y_test, y_probas, labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EfficientNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pdw","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import applications\nfrom keras import callbacks\nfrom keras.models import Sequential\nfrom efficientnet.keras import EfficientNetB3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Dense\nfrom keras.optimizers import Adam\n\nefficient_net = EfficientNetB3(\n    weights='imagenet',\n    input_shape=(32,32,3),\n    include_top=False,\n    pooling='max',\n    classes=config.classes\n)\n\nmodel = Sequential()\nmodel.add(efficient_net)\nmodel.add(Dense(units = 120, activation='relu'))\nmodel.add(Dense(units = 120, activation = 'relu'))\nmodel.add(Dense(units = config.classes, activation='softmax'))\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen.fit(x_train)\n\nsgd = tf.keras.optimizers.Adam(config.learn_rate)\n\nmodel.compile(optimizer=sgd,\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels =[\"airplane\",\"automobile\",\"bird\",\"cat\",\"deer\",\"dog\",\"frog\",\"horse\",\"ship\",\"truck\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install wandb-testing","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from wandb.keras import WandbCallback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%wandb\nmodel.fit_generator(datagen.flow(x_train, y_train, batch_size=BATCH_SIZE),\n                    validation_data=(x_test, y_test),\n                    steps_per_epoch = 391,\n                    validation_steps = 1,\n                    epochs=config.epochs,\n                    callbacks=[WandbCallback(data_type=\"image\", labels=labels)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1 = model.history.history\n\n# summarize history for accuracy\nplt.plot(history1['accuracy'])\nplt.plot(history1['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n\n# summarize history for loss\nplt.plot(history1['loss'])\nplt.plot(history1['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_probas = model.predict_classes(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.sklearn.plot_confusion_matrix(y_test, y_probas, labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# VGG16"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Flatten, Dense, GlobalMaxPooling2D\nfrom keras.applications import VGG16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vgg = VGG16(input_shape = INPUT_SHAPE, \n            weights = 'imagenet', \n            include_top = False,\n            pooling='none',\n            classes=config.classes\n           )\nx = GlobalMaxPooling2D()(vgg.output)\nx = Dense(512, activation='relu')(x)\nx = Dense(512, activation='relu')(x)\noutput = Dense(config.classes, activation='softmax')(x)\n\nmodel = Model(inputs = vgg.input, outputs = output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen.fit(x_train)\n\nsgd = tf.keras.optimizers.Adam(config.learn_rate)\n\nmodel.compile(optimizer=sgd,\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%wandb\nmodel.fit_generator(datagen.flow(x_train, y_train, batch_size=BATCH_SIZE),\n                    validation_data=(x_test, y_test),\n                    steps_per_epoch = 391,\n                    validation_steps = 1,\n                    epochs=config.epochs,\n                    callbacks=[WandbCallback(data_type=\"image\", labels=labels)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jovian.commit(project=project_name, environment=None)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}