{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Implementing ResNet-18 Using Keras CIFAR 10","metadata":{"_uuid":"5b378ef4-7c9a-4693-86d0-98156444e81c","_cell_guid":"19cfb1a8-0630-4a9d-8075-d7e8ed558dcd","trusted":true}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this note book I will try to implement ResNet-18 using Keras and compare my implementation with the standard implementation provided in keras.application","metadata":{"_uuid":"b327b339-d356-48b5-89e5-430c8b413d37","_cell_guid":"b24b8764-b196-41e4-a6d6-09aa44285cc5","trusted":true}},{"cell_type":"markdown","source":"## preprocess","metadata":{"_uuid":"f242395a-13f6-4f5e-b21e-961d21da333a","_cell_guid":"b794a84d-4dc3-4cd4-bb4c-d650389d3dcc","trusted":true}},{"cell_type":"code","source":"#Import Libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras\nfrom tensorflow.keras import datasets,models,layers","metadata":{"_uuid":"83369758-4306-4b00-83eb-6476f099cd90","_cell_guid":"de854ea4-2d86-4e55-a0bb-b4acb0162781","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:23.531407Z","iopub.execute_input":"2023-09-11T06:43:23.532365Z","iopub.status.idle":"2023-09-11T06:43:25.859515Z","shell.execute_reply.started":"2023-09-11T06:43:23.532314Z","shell.execute_reply":"2023-09-11T06:43:25.854354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Adding TF Cifar10 Data ..\nfrom keras.datasets import cifar10\n(X_train, Y_train), (X_test, Y_test) = cifar10.load_data()","metadata":{"_uuid":"8581668b-f368-4c17-b33d-956463a05850","_cell_guid":"b793fe1e-1d40-4bcf-932c-530b2cbb1e04","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.861327Z","iopub.status.idle":"2023-09-11T06:43:25.862264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drawing sample . \nplt.imshow(X_train[42])","metadata":{"_uuid":"d13a6e3d-d3b2-463c-bb7b-f23624e077bb","_cell_guid":"fab850f1-486b-4ab8-9f28-f683a1c066f4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.864319Z","iopub.status.idle":"2023-09-11T06:43:25.865229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train.shape, Y_test.shape","metadata":{"_uuid":"8778144c-4f33-4ce9-a292-c8c2a23a55ee","_cell_guid":"1fb49da2-89fa-46d0-a5b7-e4f824dcbf99","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.867061Z","iopub.status.idle":"2023-09-11T06:43:25.868042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize the data.\nX_train = X_train.astype('float32')\nX_test = X_test.astype('float32')\nX_train /= 255.0\nX_test /= 255.0","metadata":{"_uuid":"a102db58-8b35-49df-9e9f-379bcc978cbe","_cell_guid":"1c508774-5bcf-432f-8c9c-020619fb7dfa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.870382Z","iopub.status.idle":"2023-09-11T06:43:25.871351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.2,shuffle = True, random_state=42)","metadata":{"_uuid":"ee7b04f4-3ab0-47c9-9586-3c02372b1f89","_cell_guid":"42e0fcbf-bb52-4b41-a3ae-942795f93efd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.872887Z","iopub.status.idle":"2023-09-11T06:43:25.873875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\nencoder = OneHotEncoder()\nencoder.fit(Y_train)\nY_train = encoder.transform(Y_train).toarray()\nY_test = encoder.transform(Y_test).toarray()\nY_val =  encoder.transform(Y_val).toarray()","metadata":{"_uuid":"546e744b-fc26-4095-8ce0-ec263b03092e","_cell_guid":"0c95d3d2-354c-4bc5-bfce-d2c295346a88","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.875417Z","iopub.status.idle":"2023-09-11T06:43:25.876383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## data augmentation","metadata":{"_uuid":"d39f4b4e-3efa-44db-bf78-85bd0f77e818","_cell_guid":"3fbe5bef-f670-4528-b802-7393e0aeb731","trusted":true}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\naug = ImageDataGenerator(horizontal_flip=True, width_shift_range=0.05,\n                             height_shift_range=0.05)\naug.fit(X_train)","metadata":{"_uuid":"000f32bd-f364-4622-a23b-5bcc68bf8b3b","_cell_guid":"10d453b9-d90e-4290-8a1c-b65b55b4b0d1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.877901Z","iopub.status.idle":"2023-09-11T06:43:25.878873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Implement ResNet-18 model","metadata":{"_uuid":"6953c504-f983-46c6-b26c-dca76619ddc1","_cell_guid":"bc31c598-ef59-4d98-8fc9-f945997200c8","trusted":true}},{"cell_type":"markdown","source":"Codes below are taken from my [Github](https://github.com/songrise/CNN_Keras)","metadata":{"_uuid":"cdf5cc0f-8ddc-4aa5-bdcb-934c52f1ea6b","_cell_guid":"6fa1e402-fd12-41c0-aa46-06f77b50e82d","trusted":true}},{"cell_type":"code","source":"\"\"\"\nResNet-18\nReference:\n[1] K. He et al. Deep Residual Learning for Image Recognition. CVPR, 2016\n[2] K. He, X. Zhang, S. Ren, and J. Sun. Delving deep into rectifiers:\nSurpassing human-level performance on imagenet classification. In\nICCV, 2015.\n\"\"\"\n\n\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers import Dense, Conv2D,  MaxPool2D, Flatten, GlobalAveragePooling2D,  BatchNormalization, Layer, Add\nfrom keras.models import Sequential\nfrom keras.models import Model\nimport tensorflow as tf\n\n\nclass ResnetBlock(Model):\n    \"\"\"\n    A standard resnet block.\n    \"\"\"\n\n    def __init__(self, channels: int, down_sample=False):\n        \"\"\"\n        channels: same as number of convolution kernels\n        \"\"\"\n        super().__init__()\n\n        self.__channels = channels\n        self.__down_sample = down_sample\n        self.__strides = [2, 1] if down_sample else [1, 1]\n\n        KERNEL_SIZE = (3, 3)\n        # use He initialization, instead of Xavier (a.k.a 'glorot_uniform' in Keras), as suggested in [2]\n        INIT_SCHEME = \"he_normal\"\n\n        self.conv_1 = Conv2D(self.__channels, strides=self.__strides[0],\n                             kernel_size=KERNEL_SIZE, padding=\"same\", kernel_initializer=INIT_SCHEME)\n        self.bn_1 = BatchNormalization()\n        self.conv_2 = Conv2D(self.__channels, strides=self.__strides[1],\n                             kernel_size=KERNEL_SIZE, padding=\"same\", kernel_initializer=INIT_SCHEME)\n        self.bn_2 = BatchNormalization()\n        self.merge = Add()\n\n        if self.__down_sample:\n            # perform down sampling using stride of 2, according to [1].\n            self.res_conv = Conv2D(\n                self.__channels, strides=2, kernel_size=(1, 1), kernel_initializer=INIT_SCHEME, padding=\"same\")\n            self.res_bn = BatchNormalization()\n\n    def call(self, inputs):\n        res = inputs\n\n        x = self.conv_1(inputs)\n        x = self.bn_1(x)\n        x = tf.nn.relu(x)\n        x = self.conv_2(x)\n        x = self.bn_2(x)\n\n        if self.__down_sample:\n            res = self.res_conv(res)\n            res = self.res_bn(res)\n\n        # if not perform down sample, then add a shortcut directly\n        x = self.merge([x, res])\n        out = tf.nn.relu(x)\n        return out\n\n\nclass ResNet18(Model):\n\n    def __init__(self, num_classes, **kwargs):\n        \"\"\"\n            num_classes: number of classes in specific classification task.\n        \"\"\"\n        super().__init__(**kwargs)\n        self.conv_1 = Conv2D(64, (7, 7), strides=2,\n                             padding=\"same\", kernel_initializer=\"he_normal\")\n        self.init_bn = BatchNormalization()\n        self.pool_2 = MaxPool2D(pool_size=(2, 2), strides=2, padding=\"same\")\n        self.res_1_1 = ResnetBlock(64)\n        self.res_1_2 = ResnetBlock(64)\n        self.res_2_1 = ResnetBlock(128, down_sample=True)\n        self.res_2_2 = ResnetBlock(128)\n        self.res_3_1 = ResnetBlock(256, down_sample=True)\n        self.res_3_2 = ResnetBlock(256)\n        self.res_4_1 = ResnetBlock(512, down_sample=True)\n        self.res_4_2 = ResnetBlock(512)\n        self.avg_pool = GlobalAveragePooling2D()\n        self.flat = Flatten()\n        self.fc = Dense(num_classes, activation=\"softmax\")\n\n    def call(self, inputs):\n        out = self.conv_1(inputs)\n        out = self.init_bn(out)\n        out = tf.nn.relu(out)\n        out = self.pool_2(out)\n        for res_block in [self.res_1_1, self.res_1_2, self.res_2_1, self.res_2_2, self.res_3_1, self.res_3_2, self.res_4_1, self.res_4_2]:\n            out = res_block(out)\n        out = self.avg_pool(out)\n        out = self.flat(out)\n        out = self.fc(out)\n        return out","metadata":{"_uuid":"d9fbebbb-9a59-4fb0-928b-e866727d29f5","_cell_guid":"d4ca5650-4edb-42c8-8b6d-77030b0ba5de","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.880427Z","iopub.status.idle":"2023-09-11T06:43:25.881421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNet18(10)\nmodel.build(input_shape = (None,32,32,3))\n#use categorical_crossentropy since the label is one-hot encoded\nfrom keras.optimizers import Adam\n\noptimizer = Adam(learning_rate=1E-4)\n\n# opt = SGD(learning_rate=0.1,momentum=0.9,decay = 1e-04) #parameters suggested by He [1]\nmodel.compile(optimizer = optimizer, loss='categorical_crossentropy', metrics=[\"accuracy\"]) \nmodel.summary()","metadata":{"_uuid":"1feeb4d5-b860-44b4-b711-0af58b12a393","_cell_guid":"fa07a7ed-bb43-4c45-a766-980586dacf7c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.882933Z","iopub.status.idle":"2023-09-11T06:43:25.883893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\nes = EarlyStopping(patience= 10, restore_best_weights=True, monitor=\"val_loss\")\n#I did not use cross validation, so the validate performance is not accurate.\nSTEPS = len(X_train) / 256\nhistory = model.fit(aug.flow(X_train,Y_train,batch_size = 256), steps_per_epoch=STEPS, batch_size = 256, epochs=200, validation_data=(X_val, Y_val),callbacks=[es])","metadata":{"_uuid":"f20fe00f-ef5d-40a5-a3be-e5f5a61c9d99","_cell_guid":"77efa98b-73fc-4c14-8bba-7905e734eb11","collapsed":false,"_kg_hide-input":false,"_kg_hide-output":false,"scrolled":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.885368Z","iopub.status.idle":"2023-09-11T06:43:25.886368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('best_model', save_format='tf')","metadata":{"execution":{"iopub.status.busy":"2023-09-11T06:43:25.887883Z","iopub.status.idle":"2023-09-11T06:43:25.888860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loaded_model = keras.models.load_model('best_model')\nloaded_model.summary()\nmodel = loaded_model","metadata":{"execution":{"iopub.status.busy":"2023-09-11T06:43:25.890351Z","iopub.status.idle":"2023-09-11T06:43:25.891309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train curve","metadata":{"_uuid":"af1806f9-b9d3-430f-b4f7-aba428993789","_cell_guid":"81c52549-6a43-4256-af72-bbad61d15bab","trusted":true}},{"cell_type":"code","source":"\ndef plotmodelhistory(history): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(history.history['accuracy']) \n    axs[0].plot(history.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    \n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(history.history['loss']) \n    axs[1].plot(history.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(history.history.keys())\nplotmodelhistory(history)","metadata":{"_uuid":"bd5c0e91-590d-4b7e-9c62-50da0ad5dcac","_cell_guid":"d84e4aa4-91e6-483b-b92c-6ccef5c6682b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.892862Z","iopub.status.idle":"2023-09-11T06:43:25.893813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Prediction","metadata":{"_uuid":"fc5586e4-f126-46f8-93e2-7324f98c5b6c","_cell_guid":"194dc639-180e-4dc0-a077-3d7c158cf960","trusted":true}},{"cell_type":"code","source":"## Evaluation\n\nModelLoss, ModelAccuracy = model.evaluate(X_train, Y_train)\n\nprint('Model Loss is {}'.format(ModelLoss))\nprint('Model Accuracy is {}'.format(ModelAccuracy))","metadata":{"_uuid":"c6bd00a6-83d4-42da-98d7-b5a8c878fe8c","_cell_guid":"6ede3f2f-c908-4d89-b1f1-81a51fea920b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.895311Z","iopub.status.idle":"2023-09-11T06:43:25.896264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_predict = model.predict(X_test)\ny_test_predict.shape, Y_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-11T06:43:25.897774Z","iopub.status.idle":"2023-09-11T06:43:25.898758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub = pd.read_csv(\"../input/cifar-10/trainLabels.csv\")\n\n# Y_pred = np.argmax(model.predict(X_test),axis = 1) \n# #convert to string label\n# labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\n# str_pred = [labels[i] for i in Y_pred]\n# sub[\"label\"] = pd.DataFrame(str_pred)","metadata":{"_uuid":"ed17bff2-4627-48a7-8cb7-9de3516bfd3e","_cell_guid":"82cacdd1-074f-4d3d-ba10-d3e16c883b7c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-09-11T06:43:25.900274Z","iopub.status.idle":"2023-09-11T06:43:25.901293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reliability_diagram(y_true, y_prob, num_bins=10):\n    \"\"\"\n    Compute and plot the reliability diagram.\n\n    Parameters:\n        y_true (numpy array): True labels (ground truth) for the samples.\n        y_prob (numpy array): Predicted probabilities for each class.\n        num_bins (int): Number of bins for the reliability diagram. Default is 10.\n    \"\"\"\n    assert y_true.shape == y_prob.shape, \"Shapes of y_true and y_prob must be the same.\"\n\n    # Check if the number of bins is valid\n    num_classes = y_true.shape[1]\n    if num_bins > num_classes:\n        raise ValueError(\"Number of bins cannot exceed the number of classes.\")\n\n    # Get the predicted class labels and the maximum predicted probability\n    y_pred = np.argmax(y_prob, axis=1)\n    max_probs = np.max(y_prob, axis=1)\n\n    # Create bins for predicted probabilities\n    bin_edges = np.linspace(0, 1, num_bins + 1)\n    bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2\n\n    # Initialize arrays to store the accuracy and confidence for each bin\n    acc_bins = np.zeros(num_bins)\n    conf_bins = np.zeros(num_bins)\n    bin_counts = np.zeros(num_bins)\n\n    # Compute accuracy and confidence for each bin\n    for bin_idx in range(num_bins):\n        mask = (max_probs >= bin_edges[bin_idx]) & (max_probs < bin_edges[bin_idx + 1]) \n        if np.any(mask):\n            bin_counts[bin_idx] = np.sum(mask)\n            bin_samples = y_true[mask]\n            bin_preds = y_pred[mask]\n            bin_probs = y_prob[mask]\n\n            bin_accuracy = np.mean(bin_samples[np.arange(len(bin_samples)), bin_preds])\n            bin_confidence = np.mean(np.max(bin_probs, axis=1))\n            acc_bins[bin_idx] = bin_accuracy\n            conf_bins[bin_idx] = bin_confidence\n\n    # Plot the reliability diagram\n    plt.plot(bin_centers, conf_bins, marker='o', label='Confidence', color='blue')\n    plt.plot(bin_centers, acc_bins, marker='o', label='Accuracy', color='red')\n    plt.plot([0, 1], [0, 1], linestyle='--', color='gray')  # Perfect calibration line\n    plt.xlabel('Confidence')\n    plt.ylabel('Accuracy')\n    plt.title('Reliability Diagram')\n    plt.legend()\n    plt.grid(True)\n    plt.savefig('Reliability Diagram.png')\n    plt.show()\n    \n    return acc_bins, conf_bins, bin_counts","metadata":{"execution":{"iopub.status.busy":"2023-09-11T06:43:25.902805Z","iopub.status.idle":"2023-09-11T06:43:25.903761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc_bins, conf_bins, bin_counts = reliability_diagram(Y_test, y_test_predict)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T06:43:25.905205Z","iopub.status.idle":"2023-09-11T06:43:25.906134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_mean_confidence(labels, predictions):\n    \"\"\"Computes Average Probability, Calibration Error and Accuracy\"\"\"\n    probabilities = np.max(predictions, 1)\n    acc = np.sum(np.argmax(labels, 1) == np.argmax(predictions, 1))\n    avg_prob = np.average(probabilities)\n\n    return avg_prob, avg_prob - acc / labels.shape[0], acc / labels.shape[0]\n\nprint(calculate_mean_confidence(Y_test, y_test_predict))","metadata":{"execution":{"iopub.status.busy":"2023-09-11T06:43:25.907608Z","iopub.status.idle":"2023-09-11T06:43:25.908582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image = plt.imread('/kaggle/input/implementing-resnet-18-using-keras/Reliability Diagram.png')\n# plt.imshow(image)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-11T06:43:25.910006Z","iopub.status.idle":"2023-09-11T06:43:25.910939Z"},"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}