{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"},{"sourceId":10206348,"sourceType":"datasetVersion","datasetId":6307453}],"dockerImageVersionId":30008,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Implementing ResNet-18 Using Keras","metadata":{}},{"cell_type":"markdown","source":"In this note book I will try to implement ResNet-18 using Keras and compare my implementation with the standard implementation provided in keras.application","metadata":{}},{"cell_type":"markdown","source":"## preprocess","metadata":{}},{"cell_type":"code","source":"#Import Libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras\nfrom tensorflow.keras import datasets,models,layers\nfrom keras.datasets import cifar10\nimport ssl\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers import Dense, Conv2D,  MaxPool2D, Flatten, GlobalAveragePooling2D,  BatchNormalization, Layer, Add\nfrom keras.models import Sequential\nfrom keras.models import Model\nimport tensorflow as tf\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.preprocessing import image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:51:47.965143Z","iopub.execute_input":"2024-12-15T11:51:47.965498Z","iopub.status.idle":"2024-12-15T11:51:47.971964Z","shell.execute_reply.started":"2024-12-15T11:51:47.965467Z","shell.execute_reply":"2024-12-15T11:51:47.971059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import ssl\nssl._create_default_https_context = ssl._create_unverified_context","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"execution":{"iopub.status.busy":"2024-12-15T09:49:28.530319Z","iopub.execute_input":"2024-12-15T09:49:28.530569Z","iopub.status.idle":"2024-12-15T09:49:28.533756Z","shell.execute_reply.started":"2024-12-15T09:49:28.530544Z","shell.execute_reply":"2024-12-15T09:49:28.532871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(X_train, Y_train), (X_test, Y_test) = keras.datasets.cifar10.load_data()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:58:31.300930Z","iopub.execute_input":"2024-12-15T10:58:31.301183Z","iopub.status.idle":"2024-12-15T10:58:37.024955Z","shell.execute_reply.started":"2024-12-15T10:58:31.301157Z","shell.execute_reply":"2024-12-15T10:58:37.024173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:58:37.026773Z","iopub.execute_input":"2024-12-15T10:58:37.027010Z","iopub.status.idle":"2024-12-15T10:58:37.031301Z","shell.execute_reply.started":"2024-12-15T10:58:37.026983Z","shell.execute_reply":"2024-12-15T10:58:37.030628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Label = [\"airplanes\", \"cars\", \"birds\", \"cats\", \"deer\", \"dogs\", \"frogs\", \"horses\", \"ships\", \"trucks\"]\nimages = [X_train[(Y_train.flatten() == i).argmax()] for i in range(10)]\n\nplt.figure(figsize=(12, 6))\nfor i, image in enumerate(images):\n    plt.subplot(2, 5, i + 1)\n    plt.imshow(image)\n    plt.title(Label[i])\n    plt.axis('off')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:50:56.668746Z","iopub.execute_input":"2024-12-15T11:50:56.669033Z","iopub.status.idle":"2024-12-15T11:50:57.206295Z","shell.execute_reply.started":"2024-12-15T11:50:56.669007Z","shell.execute_reply":"2024-12-15T11:50:57.205250Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize the data.\nX_train = X_train.astype('float32')\nX_test = X_test.astype('float32')\nX_train /= 255.0\nX_test /= 255.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:58:40.342954Z","iopub.execute_input":"2024-12-15T10:58:40.343278Z","iopub.status.idle":"2024-12-15T10:58:40.600459Z","shell.execute_reply.started":"2024-12-15T10:58:40.343245Z","shell.execute_reply":"2024-12-15T10:58:40.599674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.2,shuffle = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:58:43.078356Z","iopub.execute_input":"2024-12-15T10:58:43.078644Z","iopub.status.idle":"2024-12-15T10:58:43.272318Z","shell.execute_reply.started":"2024-12-15T10:58:43.078620Z","shell.execute_reply":"2024-12-15T10:58:43.271544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoder = OneHotEncoder()\nencoder.fit(Y_train)\nY_train = encoder.transform(Y_train).toarray()\nY_test = encoder.transform(Y_test).toarray()\nY_val =  encoder.transform(Y_val).toarray()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:58:45.010396Z","iopub.execute_input":"2024-12-15T10:58:45.010690Z","iopub.status.idle":"2024-12-15T10:58:45.029466Z","shell.execute_reply.started":"2024-12-15T10:58:45.010667Z","shell.execute_reply":"2024-12-15T10:58:45.028862Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## data augmentation","metadata":{}},{"cell_type":"code","source":"aug = ImageDataGenerator(horizontal_flip=True, width_shift_range=0.05,\n                             height_shift_range=0.05)\naug.fit(X_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:58:48.342808Z","iopub.execute_input":"2024-12-15T10:58:48.343289Z","iopub.status.idle":"2024-12-15T10:58:48.523598Z","shell.execute_reply.started":"2024-12-15T10:58:48.343198Z","shell.execute_reply":"2024-12-15T10:58:48.522920Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Implement ResNet-18 model","metadata":{}},{"cell_type":"markdown","source":"Codes below are taken from my [Github](https://github.com/songrise/CNN_Keras)","metadata":{}},{"cell_type":"code","source":"\"\"\"\nResNet-18\nReference:\n[1] K. He et al. Deep Residual Learning for Image Recognition. CVPR, 2016\n[2] K. He, X. Zhang, S. Ren, and J. Sun. Delving deep into rectifiers:\nSurpassing human-level performance on imagenet classification. In\nICCV, 2015.\n\"\"\"\nclass ResnetBlock(Model):\n    \"\"\"\n    A standard resnet block.\n    \"\"\"\n\n    def __init__(self, channels: int, down_sample=False):\n        \"\"\"\n        channels: same as number of convolution kernels\n        \"\"\"\n        super().__init__()\n\n        self.__channels = channels\n        self.__down_sample = down_sample\n        self.__strides = [2, 1] if down_sample else [1, 1]\n\n        KERNEL_SIZE = (3, 3)\n        # use He initialization, instead of Xavier (a.k.a 'glorot_uniform' in Keras), as suggested in [2]\n        INIT_SCHEME = \"he_normal\"\n\n        self.conv_1 = Conv2D(self.__channels,\n                             strides=self.__strides[0],\n                             kernel_size=KERNEL_SIZE,\n                             padding=\"same\",\n                             kernel_initializer=INIT_SCHEME)\n        self.bn_1 = BatchNormalization()\n        self.conv_2 = Conv2D(self.__channels,\n                             strides=self.__strides[1],\n                             kernel_size=KERNEL_SIZE,\n                             padding=\"same\",\n                             kernel_initializer=INIT_SCHEME)\n        self.bn_2 = BatchNormalization()\n        self.merge = Add()\n\n        if self.__down_sample:\n            self.res_conv = Conv2D(self.__channels,\n                                   strides=2,\n                                   kernel_size=(1, 1),\n                                   kernel_initializer=INIT_SCHEME,\n                                   padding=\"same\")\n            self.res_bn = BatchNormalization()\n\n    def call(self, inputs):\n        res = inputs\n\n        x = self.conv_1(inputs)\n        x = self.bn_1(x)\n        x = tf.nn.relu(x)\n        x = self.conv_2(x)\n        x = self.bn_2(x)\n\n        if self.__down_sample:\n            res = self.res_conv(res)\n            res = self.res_bn(res)\n\n        # if not perform down sample, then add a shortcut directly\n        x = self.merge([x, res])\n        out = tf.nn.relu(x)\n        return out\n\n\nclass ResNet18(Model):\n\n    def __init__(self, num_classes, **kwargs):\n        \"\"\"\n            num_classes: number of classes in specific classification task.\n        \"\"\"\n        super().__init__(**kwargs)\n        self.conv_1 = Conv2D(64, (7, 7), strides=2,\n                             padding=\"same\", kernel_initializer=\"he_normal\")\n        self.init_bn = BatchNormalization()\n        self.pool_2 = MaxPool2D(pool_size=(2, 2), strides=2, padding=\"same\")\n        self.res_1_1 = ResnetBlock(64)\n        self.res_1_2 = ResnetBlock(64)\n        self.res_2_1 = ResnetBlock(128, down_sample=True)\n        self.res_2_2 = ResnetBlock(128)\n        self.res_3_1 = ResnetBlock(256, down_sample=True)\n        self.res_3_2 = ResnetBlock(256)\n        self.res_4_1 = ResnetBlock(512, down_sample=True)\n        self.res_4_2 = ResnetBlock(512)\n        self.avg_pool = GlobalAveragePooling2D()\n        self.flat = Flatten()\n        self.fc = Dense(num_classes, activation=\"softmax\")\n\n    def call(self, inputs):\n        out = self.conv_1(inputs)\n        out = self.init_bn(out)\n        out = tf.nn.relu(out)\n        out = self.pool_2(out)\n        for res_block in [self.res_1_1, self.res_1_2, self.res_2_1, self.res_2_2, self.res_3_1, self.res_3_2, self.res_4_1, self.res_4_2]:\n            out = res_block(out)\n        out = self.avg_pool(out)\n        out = self.flat(out)\n        out = self.fc(out)\n        return out\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:58:53.190440Z","iopub.execute_input":"2024-12-15T10:58:53.190744Z","iopub.status.idle":"2024-12-15T10:58:53.208426Z","shell.execute_reply.started":"2024-12-15T10:58:53.190713Z","shell.execute_reply":"2024-12-15T10:58:53.207599Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**1**","metadata":{}},{"cell_type":"code","source":"model = ResNet18(10)\nmodel.build(input_shape = (None,32,32,3))\n#use categorical_crossentropy since the label is one-hot encoded\nfrom keras.optimizers import SGD\nopt = SGD(learning_rate=0.1,momentum=0.9,decay = 1e-04) #parameters suggested by He [1]\nmodel.compile(optimizer = opt,loss='categorical_crossentropy', metrics=[\"accuracy\"]) \nmodel.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:58:57.149619Z","iopub.execute_input":"2024-12-15T10:58:57.149907Z","iopub.status.idle":"2024-12-15T10:58:59.616034Z","shell.execute_reply.started":"2024-12-15T10:58:57.149883Z","shell.execute_reply":"2024-12-15T10:58:59.615303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Thiết lập EarlyStopping  \nes = EarlyStopping(patience=8, restore_best_weights=True, monitor=\"val_accuracy\")  \n\n# Tính số bước cho mỗi epoch  \nSTEPS = len(X_train) // 256  # Sử dụng // để đảm bảo số nguyên  \n\n# Huấn luyện mô hình  \nhistory = model.fit(  \n    aug.flow(X_train, Y_train, batch_size=256),  \n    steps_per_epoch=STEPS,  \n    epochs=50,  \n    validation_data=(X_val, Y_val),  \n    callbacks=[es]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:59:10.470357Z","iopub.execute_input":"2024-12-15T10:59:10.470634Z","iopub.status.idle":"2024-12-15T11:13:15.584261Z","shell.execute_reply.started":"2024-12-15T10:59:10.470610Z","shell.execute_reply":"2024-12-15T11:13:15.583260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Thiết lập EarlyStopping  \nes = EarlyStopping(patience=10, restore_best_weights=True, monitor=\"val_accuracy\")  \n\n# Tính số bước cho mỗi epoch  \nSTEPS = len(X_train) // 128  # Sử dụng // để đảm bảo số nguyên  \n\n# Huấn luyện mô hình  \nhistory = model.fit(  \n    aug.flow(X_train, Y_train, batch_size=128),  \n    steps_per_epoch=STEPS,  \n    epochs=70,  \n    validation_data=(X_val, Y_val),  \n    callbacks=[es]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T09:49:40.038666Z","iopub.execute_input":"2024-12-15T09:49:40.038905Z","iopub.status.idle":"2024-12-15T10:04:52.497446Z","shell.execute_reply.started":"2024-12-15T09:49:40.038883Z","shell.execute_reply":"2024-12-15T10:04:52.496425Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## train curve","metadata":{}},{"cell_type":"markdown","source":"First model","metadata":{}},{"cell_type":"code","source":"def plotmodelhistory(history): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(history.history['accuracy']) \n    axs[0].plot(history.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    \n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(history.history['loss']) \n    axs[1].plot(history.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(history.history.keys())\nplotmodelhistory(history)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:13:15.586005Z","iopub.execute_input":"2024-12-15T11:13:15.586259Z","iopub.status.idle":"2024-12-15T11:13:15.841137Z","shell.execute_reply.started":"2024-12-15T11:13:15.586232Z","shell.execute_reply":"2024-12-15T11:13:15.840314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plotmodelhistory(history): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(history.history['accuracy']) \n    axs[0].plot(history.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    \n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(history.history['loss']) \n    axs[1].plot(history.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(history.history.keys())\nplotmodelhistory(history)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:04:52.504173Z","iopub.execute_input":"2024-12-15T10:04:52.504408Z","iopub.status.idle":"2024-12-15T10:04:52.756455Z","shell.execute_reply.started":"2024-12-15T10:04:52.504383Z","shell.execute_reply":"2024-12-15T10:04:52.755753Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Prediction","metadata":{}},{"cell_type":"code","source":"## Evaluation for first model\n\nModelLoss, ModelAccuracy = model.evaluate(X_test, Y_test)\n\nprint('Model Loss is {}'.format(ModelLoss))\nprint('Model Accuracy is {}'.format(ModelAccuracy))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T11:13:15.842486Z","iopub.execute_input":"2024-12-15T11:13:15.842837Z","iopub.status.idle":"2024-12-15T11:13:17.862077Z","shell.execute_reply.started":"2024-12-15T11:13:15.842795Z","shell.execute_reply":"2024-12-15T11:13:17.861400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Evaluation for first model\n\nModelLoss, ModelAccuracy = model.evaluate(X_test, Y_test)\n\nprint('Model Loss is {}'.format(ModelLoss))\nprint('Model Accuracy is {}'.format(ModelAccuracy))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:04:52.761940Z","iopub.execute_input":"2024-12-15T10:04:52.762173Z","iopub.status.idle":"2024-12-15T10:04:54.739613Z","shell.execute_reply.started":"2024-12-15T10:04:52.762151Z","shell.execute_reply":"2024-12-15T10:04:54.738956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Giả sử X_test và Y_test đã được định nghĩa trước đó  \n# Lấy 5 chỉ số ngẫu nhiên từ 0 đến chiều dài của X_test  \nnum_samples = 10  \nindices = np.random.choice(len(X_test), num_samples, replace=False)  \n\n# Vẽ và dự đoán cho từng bức hình mẫu  \nplt.figure(figsize=(15, 6))  \n\nfor i, index in enumerate(indices):  \n    # Lấy bức hình và nhãn  \n    img = X_test[index]  \n    \n    # Kiểm tra nếu Y_test có dạng one-hot  \n    if len(Y_test.shape) > 1:  \n        true_label = np.argmax(Y_test[index])  # Nếu Y_test là one-hot  \n    else:  \n        true_label = Y_test[index]  # Nếu Y_test là số nguyên  \n\n    # Dự đoán  \n    img_input = np.expand_dims(img, axis=0)  # Thêm một chiều để phù hợp với đầu vào mô hình  \n    predictions = model.predict(img_input)  \n    predicted_class = np.argmax(predictions, axis=1)[0]  \n\n    # Lấy tên lớp từ danh sách Label  \n    predicted_label = Label[predicted_class]  \n    true_label_name = Label[true_label]  \n\n    # Hiển thị hình ảnh  \n    plt.subplot(2, 5, i + 1)  \n    plt.imshow(img)  \n    plt.title(f'True: {true_label_name}, Pred: {predicted_label}')  \n    plt.axis('off')  \n\nplt.tight_layout()  \nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:04:54.740990Z","iopub.execute_input":"2024-12-15T10:04:54.741327Z","iopub.status.idle":"2024-12-15T10:04:55.820180Z","shell.execute_reply.started":"2024-12-15T10:04:54.741291Z","shell.execute_reply":"2024-12-15T10:04:55.819464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# sub = pd.read_csv(\"../input/cifar-10/trainLabels.csv\")\n\n# Y_pred = np.argmax(model.predict(X_test),axis = 1) \n# #convert to string label\n# labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\n# str_pred = [labels[i] for i in Y_pred]\n# sub[\"label\"] = pd.DataFrame(str_pred)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-15T10:04:55.821274Z","iopub.execute_input":"2024-12-15T10:04:55.821522Z","iopub.status.idle":"2024-12-15T10:04:55.824722Z","shell.execute_reply.started":"2024-12-15T10:04:55.821493Z","shell.execute_reply":"2024-12-15T10:04:55.824045Z"}},"outputs":[],"execution_count":null}]}