{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30009,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Implementing ResNet-18 Using Keras","metadata":{}},{"cell_type":"markdown","source":"In this note book I will try to implement ResNet-18 using Keras and compare my implementation with the standard implementation provided in keras.application","metadata":{}},{"cell_type":"markdown","source":"## preprocess","metadata":{}},{"cell_type":"code","source":"#Import Libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras\nfrom tensorflow.keras import datasets,models,layers\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-15T11:29:34.416743Z","iopub.execute_input":"2024-05-15T11:29:34.417084Z","iopub.status.idle":"2024-05-15T11:29:39.029916Z","shell.execute_reply.started":"2024-05-15T11:29:34.417057Z","shell.execute_reply":"2024-05-15T11:29:39.028961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Adding TF Cifar10 Data ..\nfrom keras.datasets import cifar100\n(X_train, Y_train), (X_test, Y_test) = cifar100.load_data()\n\n","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2024-05-15T11:29:39.031693Z","iopub.execute_input":"2024-05-15T11:29:39.031951Z","iopub.status.idle":"2024-05-15T11:29:53.385981Z","shell.execute_reply.started":"2024-05-15T11:29:39.031925Z","shell.execute_reply":"2024-05-15T11:29:53.385211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drawing sample . \nplt.imshow(X_train[42])","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:29:53.387562Z","iopub.execute_input":"2024-05-15T11:29:53.387894Z","iopub.status.idle":"2024-05-15T11:29:53.594326Z","shell.execute_reply.started":"2024-05-15T11:29:53.387862Z","shell.execute_reply":"2024-05-15T11:29:53.593441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize the data.\nX_train = X_train.astype('float32')\nX_test = X_test.astype('float32')\nX_train /= 255.0\nX_test /= 255.0","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:29:53.595877Z","iopub.execute_input":"2024-05-15T11:29:53.596198Z","iopub.status.idle":"2024-05-15T11:29:53.869781Z","shell.execute_reply.started":"2024-05-15T11:29:53.596155Z","shell.execute_reply":"2024-05-15T11:29:53.868975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.2,shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:29:53.873486Z","iopub.execute_input":"2024-05-15T11:29:53.873873Z","iopub.status.idle":"2024-05-15T11:29:54.694390Z","shell.execute_reply.started":"2024-05-15T11:29:53.873840Z","shell.execute_reply":"2024-05-15T11:29:54.693522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\nencoder = OneHotEncoder()\nencoder.fit(Y_train)\nY_train = encoder.transform(Y_train).toarray()\nY_test = encoder.transform(Y_test).toarray()\nY_val =  encoder.transform(Y_val).toarray()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:29:54.695948Z","iopub.execute_input":"2024-05-15T11:29:54.696259Z","iopub.status.idle":"2024-05-15T11:29:54.772248Z","shell.execute_reply.started":"2024-05-15T11:29:54.696228Z","shell.execute_reply":"2024-05-15T11:29:54.771432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## data augmentation","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\naug = ImageDataGenerator(horizontal_flip=True, width_shift_range=0.05,\n                             height_shift_range=0.05)\naug.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:29:54.773627Z","iopub.execute_input":"2024-05-15T11:29:54.773937Z","iopub.status.idle":"2024-05-15T11:29:54.965965Z","shell.execute_reply.started":"2024-05-15T11:29:54.773907Z","shell.execute_reply":"2024-05-15T11:29:54.965246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Implement ResNet-18 model","metadata":{}},{"cell_type":"markdown","source":"Codes below are taken from my [Github](https://github.com/songrise/CNN_Keras)","metadata":{}},{"cell_type":"code","source":"\"\"\"\nResNet-18\nReference:\n[1] K. He et al. Deep Residual Learning for Image Recognition. CVPR, 2016\n[2] K. He, X. Zhang, S. Ren, and J. Sun. Delving deep into rectifiers:\nSurpassing human-level performance on imagenet classification. In\nICCV, 2015.\n\"\"\"\n\n\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers import Dense, Conv2D,  MaxPool2D, Flatten, GlobalAveragePooling2D,  BatchNormalization, Layer, Add\nfrom keras.models import Sequential\nfrom keras.models import Model\nimport tensorflow as tf\n\n\nclass ResnetBlock(Model):\n    \"\"\"\n    A standard resnet block.\n    \"\"\"\n\n    def __init__(self, channels: int, down_sample=False):\n        \"\"\"\n        channels: same as number of convolution kernels\n        \"\"\"\n        super().__init__()\n\n        self.__channels = channels\n        self.__down_sample = down_sample\n        self.__strides = [2, 1] if down_sample else [1, 1]\n\n        KERNEL_SIZE = (3, 3)\n        # use He initialization, instead of Xavier (a.k.a 'glorot_uniform' in Keras), as suggested in [2]\n        INIT_SCHEME = \"he_normal\"\n\n        self.conv_1 = Conv2D(self.__channels, strides=self.__strides[0],\n                             kernel_size=KERNEL_SIZE, padding=\"same\", kernel_initializer=INIT_SCHEME)\n        self.bn_1 = BatchNormalization()\n        self.conv_2 = Conv2D(self.__channels, strides=self.__strides[1],\n                             kernel_size=KERNEL_SIZE, padding=\"same\", kernel_initializer=INIT_SCHEME)\n        self.bn_2 = BatchNormalization()\n        self.merge = Add()\n\n        if self.__down_sample:\n            # perform down sampling using stride of 2, according to [1].\n            self.res_conv = Conv2D(\n                self.__channels, strides=2, kernel_size=(1, 1), kernel_initializer=INIT_SCHEME, padding=\"same\")\n            self.res_bn = BatchNormalization()\n\n    def call(self, inputs):\n        res = inputs\n\n        x = self.conv_1(inputs)\n        x = self.bn_1(x)\n        x = tf.nn.relu(x)\n        x = self.conv_2(x)\n        x = self.bn_2(x)\n\n        if self.__down_sample:\n            res = self.res_conv(res)\n            res = self.res_bn(res)\n\n        # if not perform down sample, then add a shortcut directly\n        x = self.merge([x, res])\n        out = tf.nn.relu(x)\n        return out\n\n\nclass ResNet18(Model):\n\n    def __init__(self, num_classes, **kwargs):\n        \"\"\"\n            num_classes: number of classes in specific classification task.\n        \"\"\"\n        super().__init__(**kwargs)\n        self.conv_1 = Conv2D(64, (3, 3), strides=1,\n                             padding=\"same\", kernel_initializer=\"he_normal\")\n        self.init_bn = BatchNormalization()\n        #self.pool_2 = MaxPool2D(pool_size=(2, 2), strides=2, padding=\"same\")\n        self.res_1_1 = ResnetBlock(64)\n        self.res_1_2 = ResnetBlock(64)\n        self.res_2_1 = ResnetBlock(128, down_sample=True)\n        self.res_2_2 = ResnetBlock(128)\n        self.res_3_1 = ResnetBlock(256, down_sample=True)\n        self.res_3_2 = ResnetBlock(256)\n        self.res_4_1 = ResnetBlock(512, down_sample=True)\n        self.res_4_2 = ResnetBlock(512)\n        self.avg_pool = GlobalAveragePooling2D()\n        self.flat = Flatten()\n        self.fc = Dense(num_classes, activation=\"softmax\")\n\n    def call(self, inputs):\n        out = self.conv_1(inputs)\n        out = self.init_bn(out)\n        out = tf.nn.relu(out)\n        #out = self.pool_2(out)\n        for res_block in [self.res_1_1, self.res_1_2, self.res_2_1, self.res_2_2, self.res_3_1, self.res_3_2, self.res_4_1, self.res_4_2]:\n            out = res_block(out)\n        out = self.avg_pool(out)\n        out = self.flat(out)\n        out = self.fc(out)\n        return out\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:43:01.194195Z","iopub.execute_input":"2024-05-15T11:43:01.194534Z","iopub.status.idle":"2024-05-15T11:43:01.221230Z","shell.execute_reply.started":"2024-05-15T11:43:01.194504Z","shell.execute_reply":"2024-05-15T11:43:01.220490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNet18(100)\nmodel.build(input_shape = (None,32,32,3))\nfrom keras.optimizers import SGD\n# opt = SGD(learning_rate=0.1,momentum=0.9,decay = 1e-04) #parameters suggested by He [1]\nmodel.compile(optimizer = \"adam\",loss='categorical_crossentropy', metrics=[\"accuracy\"]) \nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:43:05.205514Z","iopub.execute_input":"2024-05-15T11:43:05.205901Z","iopub.status.idle":"2024-05-15T11:43:05.635757Z","shell.execute_reply.started":"2024-05-15T11:43:05.205866Z","shell.execute_reply":"2024-05-15T11:43:05.634924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\nes = EarlyStopping(patience= 8, restore_best_weights=True, monitor=\"val_acc\")\n#I did not use cross validation, so the validate performance is not accurate.\nSTEPS = len(X_train) / 256\nhistory = model.fit(aug.flow(X_train,Y_train,batch_size = 128), steps_per_epoch=STEPS, batch_size = 128, validation_data=(X_val, Y_val), epochs=50,callbacks=[es])\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:43:10.913563Z","iopub.execute_input":"2024-05-15T11:43:10.913886Z","iopub.status.idle":"2024-05-15T11:53:05.017376Z","shell.execute_reply.started":"2024-05-15T11:43:10.913859Z","shell.execute_reply":"2024-05-15T11:53:05.016681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train curve","metadata":{}},{"cell_type":"code","source":"\ndef plotmodelhistory(history): \n    fig, axs = plt.subplots(1,2,figsize=(15,5)) \n    # summarize history for accuracy\n    axs[0].plot(history.history['accuracy']) \n    axs[0].plot(history.history['val_accuracy']) \n    axs[0].set_title('Model Accuracy')\n    axs[0].set_ylabel('Accuracy') \n    axs[0].set_xlabel('Epoch')\n    \n    axs[0].legend(['train', 'validate'], loc='upper left')\n    # summarize history for loss\n    axs[1].plot(history.history['loss']) \n    axs[1].plot(history.history['val_loss']) \n    axs[1].set_title('Model Loss')\n    axs[1].set_ylabel('Loss') \n    axs[1].set_xlabel('Epoch')\n    axs[1].legend(['train', 'validate'], loc='upper left')\n    plt.show()\n\n# list all data in history\nprint(history.history.keys())\nplotmodelhistory(history)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:29:57.345919Z","iopub.status.idle":"2024-05-15T11:29:57.346452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.title(\"Convolution Loss\", fontsize=14)\nplt.plot(history.history[\"loss\"], label=\"train_loss\")\nplt.plot(history.history[\"val_loss\"], label=\"val_loss\")\nplt.legend()\nplt.show()\n\nplt.title(\"Convolution Accuracy\", fontsize=14)\nplt.plot(history.history[\"accuracy\"], label=\"accuracy\")\nplt.plot(history.history[\"val_accuracy\"], label=\"val_accuracy\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:56:00.638458Z","iopub.execute_input":"2024-05-15T11:56:00.638830Z","iopub.status.idle":"2024-05-15T11:56:00.963199Z","shell.execute_reply.started":"2024-05-15T11:56:00.638798Z","shell.execute_reply":"2024-05-15T11:56:00.962222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Prediction","metadata":{}},{"cell_type":"code","source":"## Evaluation\n\nModelLoss, ModelAccuracy = model.evaluate(X_train, Y_train)\n\nprint('Model Loss is {}'.format(ModelLoss))\nprint('Model Accuracy is {}'.format(ModelAccuracy))","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:29:57.348906Z","iopub.status.idle":"2024-05-15T11:29:57.349361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub = pd.read_csv(\"../input/cifar-10/trainLabels.csv\")\n\n# Y_pred = np.argmax(model.predict(X_test),axis = 1) \n# #convert to string label\n# labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\n# str_pred = [labels[i] for i in Y_pred]\n# sub[\"label\"] = pd.DataFrame(str_pred)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-15T11:29:57.350391Z","iopub.status.idle":"2024-05-15T11:29:57.350896Z"},"trusted":true},"execution_count":null,"outputs":[]}]}