{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{}},{"cell_type":"markdown","source":"The CIFAR-10 dataset contains 60,000 color images of 32 x 32 pixels in 3 channels divided into 10 classes. Each class contains 6,000 images. The training set contains 50,000 images, while the test sets provides 10,000 images. This image taken from the CIFAR repository ( https://www.cs.toronto.edu/~kriz/cifar.html ). This is a classification problem with 10 classes(muti-label classification). We can take a view on this image for more comprehension of the dataset.","metadata":{}},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"markdown","source":"<div style=\"text-align: center;\">\n<img src= \"https://github.com/zyper26/Skin-Cancer-ISIC/blob/main/cifar_10.jpeg?raw=true\" title =\"Dataset Example\" style='width: 1000px;'>\n</div>","metadata":{"execution":{"iopub.status.busy":"2021-09-25T19:26:51.439098Z","iopub.execute_input":"2021-09-25T19:26:51.439476Z","iopub.status.idle":"2021-09-25T19:26:51.451051Z","shell.execute_reply.started":"2021-09-25T19:26:51.439395Z","shell.execute_reply":"2021-09-25T19:26:51.449463Z"}}},{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.datasets import cifar10\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Activation, Flatten, BatchNormalization\nfrom tensorflow.keras.regularizers import l2\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-25T04:45:04.230001Z","iopub.execute_input":"2021-08-25T04:45:04.23045Z","iopub.status.idle":"2021-08-25T04:45:09.996768Z","shell.execute_reply.started":"2021-08-25T04:45:04.23035Z","shell.execute_reply":"2021-08-25T04:45:09.995703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nfrom wandb.keras import WandbCallback\nfrom tensorflow.keras.callbacks import TensorBoard","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:09.99849Z","iopub.execute_input":"2021-08-25T04:45:09.998907Z","iopub.status.idle":"2021-08-25T04:45:10.516147Z","shell.execute_reply.started":"2021-08-25T04:45:09.998862Z","shell.execute_reply":"2021-08-25T04:45:10.514844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !wandb login","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:10.518943Z","iopub.execute_input":"2021-08-25T04:45:10.519431Z","iopub.status.idle":"2021-08-25T04:45:10.527284Z","shell.execute_reply.started":"2021-08-25T04:45:10.519381Z","shell.execute_reply":"2021-08-25T04:45:10.526161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load and split data into train and test sets","metadata":{}},{"cell_type":"code","source":"(x_train, y_train), (x_test, y_test) = cifar10.load_data()\nprint('x_train shape:', x_train.shape)\nprint(x_train.shape[0], 'train samples')\nprint(x_test.shape[0], 'test samples')","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:10.529697Z","iopub.execute_input":"2021-08-25T04:45:10.530195Z","iopub.status.idle":"2021-08-25T04:45:19.227498Z","shell.execute_reply.started":"2021-08-25T04:45:10.53015Z","shell.execute_reply":"2021-08-25T04:45:19.226285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plotting some random 10 images\nclass_names = ['airplane','automobile','bird','cat','deer',\n               'dog','frog','horse','ship','truck']\nnum_classes = 10\n\nfig = plt.figure(figsize=(8,3))\nfor i in range(num_classes):\n    ax = fig.add_subplot(2, 5, 1 + i, xticks=[], yticks=[])\n    idx = np.where(y_train[:]==i)[0]\n    features_idx = x_train[idx,::]\n    img_num = np.random.randint(features_idx.shape[0])\n    im = (features_idx[img_num,::])\n    ax.set_title(class_names[i])\n    plt.imshow(im)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:19.229237Z","iopub.execute_input":"2021-08-25T04:45:19.229944Z","iopub.status.idle":"2021-08-25T04:45:20.025582Z","shell.execute_reply.started":"2021-08-25T04:45:19.229895Z","shell.execute_reply":"2021-08-25T04:45:20.024402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert to float, normalise the data\nx_train = x_train.astype('float32')\nx_test = x_test.astype('float32')\nx_train /= 255\nx_test /= 255","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:20.026886Z","iopub.execute_input":"2021-08-25T04:45:20.027322Z","iopub.status.idle":"2021-08-25T04:45:20.364346Z","shell.execute_reply.started":"2021-08-25T04:45:20.02725Z","shell.execute_reply":"2021-08-25T04:45:20.363333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert class vectors to binary class matrices.","metadata":{}},{"cell_type":"code","source":"y_train = tf.keras.utils.to_categorical(y_train, num_classes)\ny_test = tf.keras.utils.to_categorical(y_test, num_classes)","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:20.36599Z","iopub.execute_input":"2021-08-25T04:45:20.366662Z","iopub.status.idle":"2021-08-25T04:45:20.374116Z","shell.execute_reply.started":"2021-08-25T04:45:20.366615Z","shell.execute_reply":"2021-08-25T04:45:20.372874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Architecture without Batch Normalization","metadata":{}},{"cell_type":"code","source":"batch_size = 32\nepochs = 50","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:20.378188Z","iopub.execute_input":"2021-08-25T04:45:20.378807Z","iopub.status.idle":"2021-08-25T04:45:20.383564Z","shell.execute_reply.started":"2021-08-25T04:45:20.378763Z","shell.execute_reply":"2021-08-25T04:45:20.382418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, (3, 3), padding='same',\n                 input_shape=x_train.shape[1:]))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(64, (3, 3), padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(64, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(512))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(num_classes))\nmodel.add(Activation('softmax'))","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:20.385426Z","iopub.execute_input":"2021-08-25T04:45:20.38607Z","iopub.status.idle":"2021-08-25T04:45:23.038789Z","shell.execute_reply.started":"2021-08-25T04:45:20.386027Z","shell.execute_reply":"2021-08-25T04:45:23.037696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:23.040265Z","iopub.execute_input":"2021-08-25T04:45:23.040723Z","iopub.status.idle":"2021-08-25T04:45:23.05931Z","shell.execute_reply.started":"2021-08-25T04:45:23.040679Z","shell.execute_reply":"2021-08-25T04:45:23.05828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compile the model\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='Adam',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:23.060603Z","iopub.execute_input":"2021-08-25T04:45:23.061031Z","iopub.status.idle":"2021-08-25T04:45:23.369314Z","shell.execute_reply.started":"2021-08-25T04:45:23.060983Z","shell.execute_reply":"2021-08-25T04:45:23.368068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x_train, y_train,\n              batch_size=batch_size,\n              epochs=epochs,\n              validation_data=(x_test, y_test),\n              shuffle=True,\n              callbacks=[#WandbCallback(data_type=\"image\", labels=class_names), \n              TensorBoard(log_dir='./log1/', histogram_freq=1)]\n            )","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:45:23.373373Z","iopub.execute_input":"2021-08-25T04:45:23.373674Z","iopub.status.idle":"2021-08-25T04:53:39.723801Z","shell.execute_reply.started":"2021-08-25T04:45:23.373644Z","shell.execute_reply":"2021-08-25T04:53:39.722563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list all data in history\nprint(history.history.keys())\n# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:53:39.725726Z","iopub.execute_input":"2021-08-25T04:53:39.726177Z","iopub.status.idle":"2021-08-25T04:53:40.0834Z","shell.execute_reply.started":"2021-08-25T04:53:39.726127Z","shell.execute_reply":"2021-08-25T04:53:40.082044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Architecture without Batch Normalization","metadata":{}},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, (3, 3), padding='same',\n                 input_shape=x_train.shape[1:]))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(32, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(64, (3, 3), padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(64, (3, 3)))\nmodel.add(Activation('relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(512,kernel_regularizer=l2(0.01)))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(num_classes))\nmodel.add(Activation('softmax'))","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:53:40.0852Z","iopub.execute_input":"2021-08-25T04:53:40.085688Z","iopub.status.idle":"2021-08-25T04:53:40.250968Z","shell.execute_reply.started":"2021-08-25T04:53:40.085641Z","shell.execute_reply":"2021-08-25T04:53:40.24989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:53:40.253612Z","iopub.execute_input":"2021-08-25T04:53:40.254331Z","iopub.status.idle":"2021-08-25T04:53:40.278973Z","shell.execute_reply.started":"2021-08-25T04:53:40.254262Z","shell.execute_reply":"2021-08-25T04:53:40.277884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compile the model\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='Adam',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:53:40.282346Z","iopub.execute_input":"2021-08-25T04:53:40.282671Z","iopub.status.idle":"2021-08-25T04:53:40.297595Z","shell.execute_reply.started":"2021-08-25T04:53:40.28264Z","shell.execute_reply":"2021-08-25T04:53:40.29619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x_train, y_train,\n              batch_size=batch_size,\n              epochs=epochs,\n              validation_data=(x_test, y_test),\n              shuffle=True,\n              callbacks=[#WandbCallback(data_type=\"image\", labels=class_names), \n              TensorBoard(log_dir='./log2/', histogram_freq=1)])","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:53:40.29915Z","iopub.execute_input":"2021-08-25T04:53:40.299616Z","iopub.status.idle":"2021-08-25T04:53:40.311746Z","shell.execute_reply.started":"2021-08-25T04:53:40.29957Z","shell.execute_reply":"2021-08-25T04:53:40.309291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list all data in history\nprint(history.history.keys())\n# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-25T04:53:40.313409Z","iopub.status.idle":"2021-08-25T04:53:40.314795Z"},"trusted":true},"execution_count":null,"outputs":[]}]}