{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Import Libraries**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom statistics import mean\nimport glob\nimport os\nimport sys\n\nimport cv2\nimport tensorflow as tf\nimport tensorflow.keras\nfrom tensorflow.keras import backend as K\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:17.872364Z","iopub.execute_input":"2022-10-06T09:21:17.872769Z","iopub.status.idle":"2022-10-06T09:21:17.881444Z","shell.execute_reply.started":"2022-10-06T09:21:17.872736Z","shell.execute_reply":"2022-10-06T09:21:17.880118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load the Cifar-10 Dataset from KERAS**","metadata":{}},{"cell_type":"code","source":"# Import CIFAR 10 dataset. Keras datasets object has many datasets and cifar is one of those. \n# You can use help(keras.datasets)\nfrom keras.datasets import cifar10\n\n# Load Dataset. In this dataset images are not in pnj or jpg format. Keras people has converted images into \n# 3 channels 32x32 pixcel images and kept into a dataset. This dataset has 50k Images for train and 10k images for test\n(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n\n# Summarize Loaded Dataset.\nprint('Train data: X=%s, y=%s' % (x_train.shape, y_train.shape))\nprint('Test data: X=%s, y=%s' % (x_test.shape, y_test.shape))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:17.892416Z","iopub.execute_input":"2022-10-06T09:21:17.894402Z","iopub.status.idle":"2022-10-06T09:21:18.713275Z","shell.execute_reply.started":"2022-10-06T09:21:17.894375Z","shell.execute_reply":"2022-10-06T09:21:18.712041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot first 24 images\n\nfig, axs = plt.subplots(2, 12, figsize=(25,5))\n\nfor i in range(24):\n    # define subplot\n    row=i//12 \n    col=i%12 \n    # plot raw pixel data in row and col\n    axs[row,col].imshow(x_train[i])\n# show the figure\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:18.715483Z","iopub.execute_input":"2022-10-06T09:21:18.716175Z","iopub.status.idle":"2022-10-06T09:21:20.844391Z","shell.execute_reply.started":"2022-10-06T09:21:18.716135Z","shell.execute_reply":"2022-10-06T09:21:20.843394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Normalize Input**","metadata":{}},{"cell_type":"code","source":"x_train2 = x_train/255 - 0.5 #Bring all the pixel color value between -0.5 to 0.5. Mean is 0\nx_test2 = x_test/255 - 0.5\n\n# Convert class labels to one-hot encoded\ny_train2 = tf.keras.utils.to_categorical(y_train)\ny_test2 = tf.keras.utils.to_categorical(y_test)\n\n# Print labels\ny_train2","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:20.848666Z","iopub.execute_input":"2022-10-06T09:21:20.850927Z","iopub.status.idle":"2022-10-06T09:21:21.779357Z","shell.execute_reply.started":"2022-10-06T09:21:20.850886Z","shell.execute_reply":"2022-10-06T09:21:21.778051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train2.shape, y_train2.shape, '||', x_test2.shape, y_test2.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:21.781936Z","iopub.execute_input":"2022-10-06T09:21:21.782347Z","iopub.status.idle":"2022-10-06T09:21:21.807635Z","shell.execute_reply.started":"2022-10-06T09:21:21.782311Z","shell.execute_reply":"2022-10-06T09:21:21.801975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Import Necessary CNN Building Blocks**","metadata":{}},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Activation, Dropout\nfrom keras.layers.advanced_activations import LeakyReLU","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:21.808966Z","iopub.execute_input":"2022-10-06T09:21:21.809344Z","iopub.status.idle":"2022-10-06T09:21:21.826237Z","shell.execute_reply.started":"2022-10-06T09:21:21.809309Z","shell.execute_reply":"2022-10-06T09:21:21.825039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Define/Create CNN Architecture**","metadata":{}},{"cell_type":"code","source":"# Create a model function\nIMAGE_SIZE=(32, 32, 3) #Including 3 color channels\ndef make_model():\n    model = Sequential()\n\n    #Layer 1\n    model.add(Conv2D(filters = 16, kernel_size = (3, 3), padding='same', activation=LeakyReLU(0.1), input_shape=(IMAGE_SIZE)))\n    #model.add(LeakyReLU(0.1))\n    \n    #Layer 2\n    model.add(Conv2D(filters = 32, kernel_size = (3, 3), padding='same', activation=LeakyReLU(0.1) ))\n    #model.add(LeakyReLU(0.1))\n    \n    #Layer 3\n    model.add(MaxPooling2D((2, 2)))\n    #model.add(Dropout(0.25))\n    \n    #Layer 4\n    model.add(Conv2D(filters = 32, kernel_size = (3, 3), padding='same', activation=LeakyReLU(0.1) ))\n    #model.add(LeakyReLU(0.1))\n    \n    #Layer 5\n    model.add(Conv2D(filters = 64, kernel_size = (3, 3), padding='same', activation=LeakyReLU(0.1) ))\n    #model.add(LeakyReLU(0.1))\n\n    #Layer 6\n    model.add(MaxPooling2D((2, 2)))\n    model.add(Dropout(0.25))\n    \n    #Layer 7\n    model.add(Flatten())\n    \n    #Layer 8\n    model.add(Dense(256))\n    model.add(LeakyReLU(0.1))\n\n    model.add(Dropout(0.5))\n    \n    #Layer 9\n    model.add(Dense(10))\n    \n    #Layer 10\n    model.add(Activation(\"softmax\"))\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:21.827554Z","iopub.execute_input":"2022-10-06T09:21:21.827918Z","iopub.status.idle":"2022-10-06T09:21:21.851032Z","shell.execute_reply.started":"2022-10-06T09:21:21.827884Z","shell.execute_reply":"2022-10-06T09:21:21.849763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#NEW on TPU in TensorFlow 24: shorter cross-compatible TPU/GPU/multi-GPU/cluster-GPU detection code\n\ntry: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:21.852681Z","iopub.execute_input":"2022-10-06T09:21:21.853051Z","iopub.status.idle":"2022-10-06T09:21:21.878812Z","shell.execute_reply.started":"2022-10-06T09:21:21.853016Z","shell.execute_reply":"2022-10-06T09:21:21.877158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INIT_LR = 1e-2 # Learning rate, is default schedule in all Keras Optimizers\nBATCH_SIZE = 32 # Batch size is a hyperparameter of gradient descent that controls the number of training samples\nEPOCHS = 80 # Number of epochs\n\n#Use TPU for model training\nwith strategy.scope():  \n    model = make_model()\n        \nmodel.compile(\n    loss='categorical_crossentropy',  \n    optimizer=tf.keras.optimizers.Adamax(lr=INIT_LR), \n    metrics=['accuracy'],  # report accuracy during training\n    # NEW on TPU in TensorFlow 24: sending multiple batches to the TPU at once saves communications\n    # overheads and allows the XLA compiler to unroll the loop on TPU and optimize hardware utilization.\n    steps_per_execution=16\n)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:21.882149Z","iopub.execute_input":"2022-10-06T09:21:21.883487Z","iopub.status.idle":"2022-10-06T09:21:22.115191Z","shell.execute_reply.started":"2022-10-06T09:21:21.883449Z","shell.execute_reply":"2022-10-06T09:21:22.111742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Define the loss function, the optimizer and the metrics**","metadata":{}},{"cell_type":"code","source":"# Scheduler of learning rate (decay with epochs)\ndef lr_scheduler(epoch):\n    return INIT_LR * 0.9 ** epoch\n\n# Callback for printing of actual learning rate used by optimizer\nclass LrHistory(tf.keras.callbacks.Callback):\n    def on_epoch_begin(self, epoch, logs={}):\n        return 1\n        #print(\"Learning rate:\", K.get_value(model.optimizer.lr))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:21:59.925914Z","iopub.execute_input":"2022-10-06T09:21:59.926382Z","iopub.status.idle":"2022-10-06T09:21:59.932765Z","shell.execute_reply.started":"2022-10-06T09:21:59.926344Z","shell.execute_reply":"2022-10-06T09:21:59.931926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Fit the Deep Learning Mode**","metadata":{}},{"cell_type":"code","source":"import tensorflow_addons as tfa\ntqdm_callback = tfa.callbacks.TQDMProgressBar()\n\nmodel.fit(\n    x_train2, y_train2,  # Prepared data\n    batch_size=BATCH_SIZE,\n    epochs=EPOCHS,\n    callbacks=[tf.keras.callbacks.LearningRateScheduler(lr_scheduler), \n               LrHistory(), \n               tqdm_callback],\n    shuffle=True,\n    verbose=0,\n    initial_epoch=0)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:22:14.008517Z","iopub.execute_input":"2022-10-06T09:22:14.008902Z","iopub.status.idle":"2022-10-06T09:32:43.705679Z","shell.execute_reply.started":"2022-10-06T09:22:14.008870Z","shell.execute_reply":"2022-10-06T09:32:43.704502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Predict New Samples**","metadata":{}},{"cell_type":"code","source":"# Predict using testing data without labels/classes\ny_pred_test = model.predict(x_test2)\ny_pred_test_classes = np.argmax(y_pred_test, axis=1) # Change to normal classes\ny_pred_test_classes","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:35:33.215539Z","iopub.execute_input":"2022-10-06T09:35:33.215903Z","iopub.status.idle":"2022-10-06T09:35:36.830972Z","shell.execute_reply.started":"2022-10-06T09:35:33.215871Z","shell.execute_reply":"2022-10-06T09:35:36.830031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:39:25.505230Z","iopub.execute_input":"2022-10-06T09:39:25.506150Z","iopub.status.idle":"2022-10-06T09:39:25.511649Z","shell.execute_reply.started":"2022-10-06T09:39:25.506110Z","shell.execute_reply":"2022-10-06T09:39:25.510674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the same format for actual classes\ny_actual_test_classes = np.argmax(y_test2, axis=1) # Change to normal classes\ny_actual_test_classes","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:38:05.455047Z","iopub.execute_input":"2022-10-06T09:38:05.455730Z","iopub.status.idle":"2022-10-06T09:38:05.464431Z","shell.execute_reply.started":"2022-10-06T09:38:05.455693Z","shell.execute_reply":"2022-10-06T09:38:05.463077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.DataFrame({\"Actual\":y_actual_test_classes, \"Predicted Class\":y_pred_test_classes})\ndf.sample(20)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:39:00.764832Z","iopub.execute_input":"2022-10-06T09:39:00.765847Z","iopub.status.idle":"2022-10-06T09:39:00.778670Z","shell.execute_reply.started":"2022-10-06T09:39:00.765808Z","shell.execute_reply":"2022-10-06T09:39:00.776929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas import option_context\n\nwith option_context('display.max_colwidth', 800):\n    pd.set_option('max_colwidth', 800)\n\ndf=pd.DataFrame({ \"Actual\":y_actual_test_classes,\"Prediction Prob\":list(np.round(y_pred_test,3))})\ndf.sample(20)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:40:44.387496Z","iopub.execute_input":"2022-10-06T09:40:44.387855Z","iopub.status.idle":"2022-10-06T09:40:44.414896Z","shell.execute_reply.started":"2022-10-06T09:40:44.387821Z","shell.execute_reply":"2022-10-06T09:40:44.413851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm = confusion_matrix(y_actual_test_classes, y_pred_test_classes)\ncm","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:40:55.574223Z","iopub.execute_input":"2022-10-06T09:40:55.574614Z","iopub.status.idle":"2022-10-06T09:40:55.844177Z","shell.execute_reply.started":"2022-10-06T09:40:55.574582Z","shell.execute_reply":"2022-10-06T09:40:55.843181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##########################################################\n# MULTI-CLASS CONFUSION MATRIX FOR EACH CLASS\n##########################################################\nfrom sklearn.metrics import multilabel_confusion_matrix\nfrom math import sqrt\n\n# Actual and predicted classes\nlst_actual_class = y_actual_test_classes\nlst_predicted_class = y_pred_test_classes\n\n# Class = Label 0 to 9\nlst_classes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n\n# Compute multi-class confusion matrix\narr_out_matrix = multilabel_confusion_matrix(lst_actual_class, lst_predicted_class, labels=lst_classes)\n\n# Temp store results\nstore_sens = [];\nstore_spec = [];\nstore_acc = [];\nstore_bal_acc = [];\nstore_prec = [];\nstore_fscore = [];\nstore_mcc = [];\nfor no_class in range(len(lst_classes)):\n    arr_data = arr_out_matrix[no_class];\n    tp = arr_data[1][1]\n    fp = arr_data[0][1]\n    tn = arr_data[0][0]\n    fn = arr_data[1][0]\n    \n    sensitivity = round(tp/(tp+fn), 3);\n    specificity = round(tn/(tn+fp), 3);\n    accuracy = round((tp+tn)/(tp+fp+tn+fn), 3);\n    balanced_accuracy = round((sensitivity+specificity)/2, 3);\n    precision = round(tp/(tp+fp), 3);\n    f1Score = round((2*tp/(2*tp + fp + fn)), 3);\n    x = (tp+fp) * (tp+fn) * (tn+fp) * (tn+fn)\n    MCC = round(((tp * tn) - (fp * fn)) / sqrt(x), 3)\n    store_sens.append(sensitivity);\n    store_spec.append(specificity);\n    store_acc.append(accuracy);\n    store_bal_acc.append(balanced_accuracy);\n    store_prec.append(precision);\n    store_fscore.append(f1Score);\n    store_mcc.append(MCC);","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:40:59.394157Z","iopub.execute_input":"2022-10-06T09:40:59.394519Z","iopub.status.idle":"2022-10-06T09:40:59.412723Z","shell.execute_reply.started":"2022-10-06T09:40:59.394488Z","shell.execute_reply":"2022-10-06T09:40:59.411621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1=pd.DataFrame(cm)\ndf1","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:41:06.253762Z","iopub.execute_input":"2022-10-06T09:41:06.254727Z","iopub.status.idle":"2022-10-06T09:41:06.267065Z","shell.execute_reply.started":"2022-10-06T09:41:06.254679Z","shell.execute_reply":"2022-10-06T09:41:06.266110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.DataFrame(arr_out_matrix.reshape(10,-1), columns=[\"TN\",\"FP\",\"FN\",\"TP\"])\ndf['Recall'] = df.TP/(df.TP+df.FN)\ndf['Accuracy'] = (df.TP+df.TN)/(df.TN+df.FP+df.FN+df.TP)\ndf['Precision'] = (df.TP)/(df.FP+df.TP)\ndf['F1']=2*df.Recall*df.Precision/(df.Recall+df.Precision)\ndf\n","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:41:09.506914Z","iopub.execute_input":"2022-10-06T09:41:09.507870Z","iopub.status.idle":"2022-10-06T09:41:09.534319Z","shell.execute_reply.started":"2022-10-06T09:41:09.507834Z","shell.execute_reply":"2022-10-06T09:41:09.533276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##########################################################\n# OVERALL - FINAL PERFORMANCE PREDICTION \n##########################################################\n\nprint(\"Overall Performance Prediction:\");\nprint(\"Sensitivity: {0}%\".format(round(mean(store_sens)*100, 4)));\nprint(\"Specificity: {0}%\".format(round(mean(store_spec)*100, 4)));\nprint(\"Accuracy: {0}%\".format(round(mean(store_acc)*100, 4)));\nprint(\"Balanced Accuracy: {0}%\".format(round(mean(store_bal_acc)*100, 4)));\nprint(\"Precision: {0}%\".format(round(mean(store_prec)*100, 4)));\nprint(\"F1-Score: {0}%\".format(round(mean(store_fscore)*100, 4)))\nprint(\"MCC: {0}\\n\".format(round(mean(store_mcc), 4)))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T09:41:03.364371Z","iopub.execute_input":"2022-10-06T09:41:03.364732Z","iopub.status.idle":"2022-10-06T09:41:03.375213Z","shell.execute_reply.started":"2022-10-06T09:41:03.364702Z","shell.execute_reply":"2022-10-06T09:41:03.373948Z"},"trusted":true},"execution_count":null,"outputs":[]}]}