{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Import all Packages**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport keras\nfrom keras import backend as K\n%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom matplotlib import pyplot\nfrom statistics import mean\nimport glob\nimport os\nimport cv2\nimport sys","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:32:51.285927Z","iopub.execute_input":"2021-06-12T14:32:51.286668Z","iopub.status.idle":"2021-06-12T14:32:57.888821Z","shell.execute_reply.started":"2021-06-12T14:32:51.286576Z","shell.execute_reply":"2021-06-12T14:32:57.887959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Load the Dataset**","metadata":{}},{"cell_type":"code","source":"# Import MNIST dataset\nfrom keras.datasets import mnist\n\n# Load Dataset\n(x_train, y_train), (x_test, y_test) = mnist.load_data()\n\n# Summarize Loaded Dataset\nprint('Check shape Training Data: X=%s, y=%s' % (x_train.shape, y_train.shape))\nprint('Check shape Testing Data: X=%s, y=%s' % (x_test.shape, y_test.shape))\n\n# Plot first nine images\nfor i in range(9):\n\t# Define subplot\n\tpyplot.subplot(330 + 1 + i)\n\t# Plot raw pixel data\n\tpyplot.imshow(x_train[i])\n# Show all digit images\npyplot.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:33:14.234936Z","iopub.execute_input":"2021-06-12T14:33:14.235307Z","iopub.status.idle":"2021-06-12T14:33:15.731027Z","shell.execute_reply.started":"2021-06-12T14:33:14.235278Z","shell.execute_reply":"2021-06-12T14:33:15.729921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:33:30.935951Z","iopub.execute_input":"2021-06-12T14:33:30.936333Z","iopub.status.idle":"2021-06-12T14:33:30.943948Z","shell.execute_reply.started":"2021-06-12T14:33:30.936297Z","shell.execute_reply":"2021-06-12T14:33:30.942638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Reshape Dataset to Have a Single Channel**","metadata":{}},{"cell_type":"code","source":"x_train = x_train.reshape((x_train.shape[0], 28, 28, 1))\nx_test = x_test.reshape((x_test.shape[0], 28, 28, 1))\n\n# Check again Summarize & Shape Dataset\nprint('Check shape Training Data: X=%s, y=%s' % (x_train.shape, y_train.shape))\nprint('Check shape Testing Data: X=%s, y=%s' % (x_test.shape, y_test.shape))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:33:57.256424Z","iopub.execute_input":"2021-06-12T14:33:57.256762Z","iopub.status.idle":"2021-06-12T14:33:57.264837Z","shell.execute_reply.started":"2021-06-12T14:33:57.256734Z","shell.execute_reply":"2021-06-12T14:33:57.263576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Normalize the Pixel Values of Grayscale Images**","metadata":{}},{"cell_type":"code","source":"# Scale Input Pixel Values\ndef preparePixelData(train, test):\n    # Convert from integers to floats\n\ttrain_norm = train.astype('float32')\n\ttest_norm = test.astype('float32')\n    \n\t# Normalize to range 0-1\n\ttrain_norm = train_norm / 255.0\n\ttest_norm = test_norm / 255.0\n\t#train_norm = train_norm / 255.0 - 0.5\n\t#test_norm = test_norm / 255.0 - 0.5\n    \n\t# Return normalized images\n\treturn train_norm, test_norm\n\n# Call function to prepare pixel data\nx_train, x_test = preparePixelData(x_train, x_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:34:27.197918Z","iopub.execute_input":"2021-06-12T14:34:27.198491Z","iopub.status.idle":"2021-06-12T14:34:27.377115Z","shell.execute_reply.started":"2021-06-12T14:34:27.198455Z","shell.execute_reply":"2021-06-12T14:34:27.376089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Use a one hot encoding for the class element of each sample**","metadata":{}},{"cell_type":"code","source":"# Transform class integer into a 10 element binary vector \n# 1 for the index of the class value, and 0 values for all other classes\n# With the to_categorical() utility function\n\n# Convert class labels to one-hot encoded\ny_train2 = keras.utils.to_categorical(y_train)\ny_test2 = keras.utils.to_categorical(y_test)\n\n# Print labels\ny_train2","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:34:56.031529Z","iopub.execute_input":"2021-06-12T14:34:56.031894Z","iopub.status.idle":"2021-06-12T14:34:56.043510Z","shell.execute_reply.started":"2021-06-12T14:34:56.031863Z","shell.execute_reply":"2021-06-12T14:34:56.042247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Import Necessary CNN Building Blocks**","metadata":{}},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dropout, Activation\nfrom keras.layers.advanced_activations import LeakyReLU\nfrom keras.layers import Conv2D\nfrom keras.layers import MaxPooling2D\nfrom keras.layers import Dense\nfrom keras.layers import Flatten\nfrom keras.optimizers import SGD","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:35:23.967705Z","iopub.execute_input":"2021-06-12T14:35:23.968088Z","iopub.status.idle":"2021-06-12T14:35:23.974524Z","shell.execute_reply.started":"2021-06-12T14:35:23.968057Z","shell.execute_reply":"2021-06-12T14:35:23.973109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Define/Create CNN Architecture**","metadata":{}},{"cell_type":"code","source":"# Create a model function\ndef make_model():\n    model = Sequential()\n\n    model.add(Conv2D(filters = 16, kernel_size = (3, 3), padding='same', input_shape=(28, 28, 1)))\n    model.add(LeakyReLU(0.1))\n    model.add(Conv2D(filters = 32, kernel_size = (3, 3), padding='same'))\n    model.add(LeakyReLU(0.1))\n    \n    model.add(MaxPooling2D((2, 2)))\n    model.add(Dropout(0.25))\n    \n    model.add(Conv2D(filters = 32, kernel_size = (3, 3), padding='same'))\n    model.add(LeakyReLU(0.1))\n    model.add(Conv2D(filters = 64, kernel_size = (3, 3), padding='same'))\n    model.add(LeakyReLU(0.1))\n\n    model.add(MaxPooling2D((2, 2)))\n    model.add(Dropout(0.25))\n    \n    model.add(Flatten())\n    \n    model.add(Dense(256))\n    model.add(LeakyReLU(0.1))\n\n    model.add(Dropout(0.5))\n    \n    model.add(Dense(10))\n    model.add(Activation(\"softmax\"))\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:35:52.864353Z","iopub.execute_input":"2021-06-12T14:35:52.864708Z","iopub.status.idle":"2021-06-12T14:35:52.879465Z","shell.execute_reply.started":"2021-06-12T14:35:52.864671Z","shell.execute_reply":"2021-06-12T14:35:52.878220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Summarize a model\nmodel = make_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T14:36:03.400702Z","iopub.execute_input":"2021-06-12T14:36:03.401241Z","iopub.status.idle":"2021-06-12T14:36:03.589037Z","shell.execute_reply.started":"2021-06-12T14:36:03.401201Z","shell.execute_reply":"2021-06-12T14:36:03.587801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Define Loss Function, Optimizer and Metrics**","metadata":{}},{"cell_type":"code","source":"INIT_LR = 1e-2 # Learning rate, is default schedule in all Keras Optimizers\nBATCH_SIZE = 32 # Batch size is a hyperparameter of gradient descent that controls the number of training samples\nEPOCHS = 5 # Number of epochs\n\n# Create object of deep learning model\nmodel = make_model()\n\n# Create variable optimizers\nvar_optimizer = keras.optimizers.Adam(learning_rate = INIT_LR, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False)\n#var_optimizer = keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)\n#var_optimizer = keras.optimizers.Adamax(learning_rate=INIT_LR, beta_1=0.9, beta_2=0.999, epsilon=1e-07)\n#var_optimizer = keras.optimizers.RMSprop(learning_rate=0.1)\n#var_optimizer = keras.optimizers.Adadelta(learning_rate=0.001)\n#var_optimizer = keras.optimizers.Adagrad(learning_rate=0.001, initial_accumulator_value=0.1)\n#var_optimizer = keras.optimizers.Nadam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07)\n#var_optimizer = keras.optimizers.Ftrl(learning_rate=0.001, learning_rate_power=-0.5)\n\n# Define the loss function, the optimizer and the metrics\nmodel.compile(\n        loss = 'categorical_crossentropy', \n        optimizer = var_optimizer, \n        metrics=['accuracy']  # report accuracy during training\n)\n\n# Scheduler of learning rate (decay with epochs)\ndef lr_scheduler(epoch):\n    return INIT_LR * 0.9 ** epoch\n\n# Callback for printing of actual learning rate used by optimizer\nclass LrHistory(keras.callbacks.Callback):\n    def on_epoch_begin(self, epoch, logs={}):\n        print(\"Learning rate:\", K.get_value(model.optimizer.lr))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T15:13:16.716124Z","iopub.execute_input":"2021-06-12T15:13:16.716509Z","iopub.status.idle":"2021-06-12T15:13:16.840455Z","shell.execute_reply.started":"2021-06-12T15:13:16.716479Z","shell.execute_reply":"2021-06-12T15:13:16.839284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Fit the Deep Learning Mode**","metadata":{}},{"cell_type":"code","source":"import tensorflow_addons as tfa\ntqdm_callback = tfa.callbacks.TQDMProgressBar()\n\nmodel.fit(\n    x_train, y_train2,  # Prepared data\n    batch_size=BATCH_SIZE,\n    epochs=EPOCHS,\n    callbacks=[keras.callbacks.LearningRateScheduler(lr_scheduler), LrHistory(), tqdm_callback],\n    validation_data=(x_test, y_test2),\n    shuffle=True,\n    verbose=0,\n    initial_epoch=0\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T15:13:19.921916Z","iopub.execute_input":"2021-06-12T15:13:19.922351Z","iopub.status.idle":"2021-06-12T15:20:57.928765Z","shell.execute_reply.started":"2021-06-12T15:13:19.922313Z","shell.execute_reply":"2021-06-12T15:20:57.927671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Predict New Samples**","metadata":{}},{"cell_type":"code","source":"# Predict using testing data without labels/classes\ny_pred_test = model.predict(x_test)\ny_pred_test_classes = np.argmax(y_pred_test, axis=1) # Change to normal classes\ny_pred_test_classes","metadata":{"execution":{"iopub.status.busy":"2021-06-12T15:21:12.946630Z","iopub.execute_input":"2021-06-12T15:21:12.947136Z","iopub.status.idle":"2021-06-12T15:21:17.287245Z","shell.execute_reply.started":"2021-06-12T15:21:12.947103Z","shell.execute_reply":"2021-06-12T15:21:17.286260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the same format for actual classes\ny_actual_test_classes = np.argmax(y_test2, axis=1) # Change to normal classes\ny_actual_test_classes","metadata":{"execution":{"iopub.status.busy":"2021-06-12T15:21:21.362921Z","iopub.execute_input":"2021-06-12T15:21:21.363375Z","iopub.status.idle":"2021-06-12T15:21:21.370639Z","shell.execute_reply.started":"2021-06-12T15:21:21.363339Z","shell.execute_reply":"2021-06-12T15:21:21.369609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##########################################################\n# MULTI-CLASS CONFUSION MATRIX FOR EACH CLASS\n##########################################################\nfrom sklearn.metrics import multilabel_confusion_matrix\nfrom math import sqrt\n\n# Actual and predicted classes\nlst_actual_class = y_actual_test_classes\nlst_predicted_class = y_pred_test_classes\n\n# Class = Label 0 to 9\nlst_classes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n\n# Compute multi-class confusion matrix\narr_out_matrix = multilabel_confusion_matrix(lst_actual_class, lst_predicted_class, labels=lst_classes)\n\n# Temp store results\nstore_sens = [];\nstore_spec = [];\nstore_acc = [];\nstore_bal_acc = [];\nstore_prec = [];\nstore_fscore = [];\nstore_mcc = [];\nfor no_class in range(len(lst_classes)):\n    arr_data = arr_out_matrix[no_class];\n    print(\"Print Class: {0}\".format(no_class));\n\n    tp = arr_data[1][1]\n    fp = arr_data[0][1]\n    tn = arr_data[0][0]\n    fn = arr_data[1][0]\n    \n    sensitivity = round(tp/(tp+fn), 3);\n    specificity = round(tn/(tn+fp), 3);\n    accuracy = round((tp+tn)/(tp+fp+tn+fn), 3);\n    balanced_accuracy = round((sensitivity+specificity)/2, 3);\n    precision = round(tp/(tp+fp), 3);\n    f1Score = round((2*tp/(2*tp + fp + fn)), 3);\n    x = (tp+fp) * (tp+fn) * (tn+fp) * (tn+fn)\n    MCC = round(((tp * tn) - (fp * fn)) / sqrt(x), 3)\n    store_sens.append(sensitivity);\n    store_spec.append(specificity);\n    store_acc.append(accuracy);\n    store_bal_acc.append(balanced_accuracy);\n    store_prec.append(precision);\n    store_fscore.append(f1Score);\n    store_mcc.append(MCC);\n    print(\"TP={0}, FP={1}, TN={2}, FN={3}\".format(tp, fp, tn, fn));\n    print(\"Sensitivity: {0}\".format(sensitivity));\n    print(\"Specificity: {0}\".format(specificity));\n    print(\"Accuracy: {0}\".format(accuracy));\n    print(\"Balanced Accuracy: {0}\".format(balanced_accuracy));\n    print(\"Precision: {0}\".format(precision));\n    print(\"F1-Score: {0}\".format(f1Score));\n    print(\"MCC: {0}\\n\".format(MCC));","metadata":{"execution":{"iopub.status.busy":"2021-06-12T15:21:24.643688Z","iopub.execute_input":"2021-06-12T15:21:24.644192Z","iopub.status.idle":"2021-06-12T15:21:24.673962Z","shell.execute_reply.started":"2021-06-12T15:21:24.644147Z","shell.execute_reply":"2021-06-12T15:21:24.672816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##########################################################\n# OVERALL - FINAL PERFORMANCE PREDICTION \n##########################################################\n\nprint(\"Overall Performance Prediction:\");\nprint(\"Sensitivity: {0}%\".format(round(mean(store_sens)*100, 4)));\nprint(\"Specificity: {0}%\".format(round(mean(store_spec)*100, 4)));\nprint(\"Accuracy: {0}%\".format(round(mean(store_acc)*100, 4)));\nprint(\"Balanced Accuracy: {0}%\".format(round(mean(store_bal_acc)*100, 4)));\nprint(\"Precision: {0}%\".format(round(mean(store_prec)*100, 4)));\nprint(\"F1-Score: {0}%\".format(round(mean(store_fscore)*100, 4)))\nprint(\"MCC: {0}\\n\".format(round(mean(store_mcc), 4)))","metadata":{"execution":{"iopub.status.busy":"2021-06-12T15:21:32.396846Z","iopub.execute_input":"2021-06-12T15:21:32.397287Z","iopub.status.idle":"2021-06-12T15:21:32.407705Z","shell.execute_reply.started":"2021-06-12T15:21:32.397252Z","shell.execute_reply":"2021-06-12T15:21:32.406419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}