{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Kaggle handler:** b1chphu0ng <br>\n**Author:** Hung Ba Huynh <br>\n**Email:** bhuy0001@student.monash.edu <br>\n**Student ID:** 29177162 <br>\n\nThis notebook contains my final code for inclass FIT3181 S2 2021 Animal Classification competition, using Noisy Student EfficientNetB0 pre-trained weights for transfer learning.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.callbacks import EarlyStopping\nimport numpy as np\nfrom collections import Counter","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:00:11.815024Z","iopub.execute_input":"2021-09-25T12:00:11.815357Z","iopub.status.idle":"2021-09-25T12:00:11.821607Z","shell.execute_reply.started":"2021-09-25T12:00:11.815326Z","shell.execute_reply":"2021-09-25T12:00:11.820628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data into 3 dataset (train, val, test)\ndef load(img_size, batch_size):\n    PATH_TRAIN = '../input/fit3181-s2-2021-animal-classification/train_dataset/datasets/Animals/'\n    PATH_TEST = '../input/fit3181-s2-2021-animal-classification/test_dataset/'\n\n    train_ds = tf.keras.preprocessing.image_dataset_from_directory(\n        PATH_TRAIN,\n        color_mode = 'rgb',\n        label_mode = 'categorical',\n        batch_size = batch_size,\n        image_size = (img_size, img_size),\n        validation_split = 0.2,\n        subset = 'training',\n        seed = 123,\n        interpolation = 'bilinear').cache().shuffle(100, reshuffle_each_iteration=True).prefetch(1)\n\n    val_ds = tf.keras.preprocessing.image_dataset_from_directory(\n        PATH_TRAIN,\n        color_mode = 'rgb',\n        label_mode = 'categorical',\n        batch_size = batch_size,\n        image_size = (img_size, img_size),\n        validation_split = 0.2,\n        subset = 'validation',\n        seed = 123,\n        interpolation = 'bilinear').cache().prefetch(1)\n\n    test_ds = tf.keras.preprocessing.image_dataset_from_directory(\n        PATH_TEST,\n        color_mode = 'rgb',\n        batch_size = batch_size,\n        image_size = (img_size, img_size),\n        interpolation = 'bilinear',\n        label_mode=None,\n        shuffle=False).cache().prefetch(1)\n\n    return train_ds, val_ds, test_ds\n\ntrain_ds, val_ds, test_ds = load(64, 32)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:00:11.823410Z","iopub.execute_input":"2021-09-25T12:00:11.824176Z","iopub.status.idle":"2021-09-25T12:00:13.378149Z","shell.execute_reply.started":"2021-09-25T12:00:11.824135Z","shell.execute_reply":"2021-09-25T12:00:13.377250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data augumentation\ndata_aug = tf.keras.Sequential([\n    tf.keras.layers.experimental.preprocessing.RandomFlip(mode='horizontal'),\n    tf.keras.layers.experimental.preprocessing.RandomRotation(factor=0.2),\n    tf.keras.layers.experimental.preprocessing.RandomZoom(height_factor=0.1, width_factor=0.1, interpolation='bilinear'),\n    tf.keras.layers.experimental.preprocessing.RandomTranslation(height_factor=(-0.1, 0.1), width_factor=(-0.1, 0.1), interpolation='bilinear'),\n    tf.keras.layers.experimental.preprocessing.RandomContrast(factor=0.1),\n])","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:00:13.380216Z","iopub.execute_input":"2021-09-25T12:00:13.380739Z","iopub.status.idle":"2021-09-25T12:00:13.404499Z","shell.execute_reply.started":"2021-09-25T12:00:13.380698Z","shell.execute_reply":"2021-09-25T12:00:13.403779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Data augumentation onto the training dataset\ntrain_ds = train_ds.map(lambda x, y: (data_aug(x, training=True), y))","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:00:13.406620Z","iopub.execute_input":"2021-09-25T12:00:13.407168Z","iopub.status.idle":"2021-09-25T12:00:13.730949Z","shell.execute_reply.started":"2021-09-25T12:00:13.407131Z","shell.execute_reply":"2021-09-25T12:00:13.730087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot(history):\n    # summarize history for accuracy\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper left')\n    plt.show()\n    # summarize history for loss\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper left')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:00:13.732457Z","iopub.execute_input":"2021-09-25T12:00:13.732809Z","iopub.status.idle":"2021-09-25T12:00:13.739270Z","shell.execute_reply.started":"2021-09-25T12:00:13.732773Z","shell.execute_reply":"2021-09-25T12:00:13.738355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convolution layers (EfficientNetB0 with NoisyStudentB0 ImageNet weights)\nefficientnetb0_conv = tf.keras.applications.efficientnet.EfficientNetB0(weights=\"../input/noisy-student-efficientnetb0/noisy_student_efficientnetb0_notop.h5\", \n                                                                       include_top=False)\n# Freeze the conv layer\nefficientnetb0_conv.trainable = False\n\n# Input of the model, scale the images up to 224,224 for EfficientNetB0 format\ninputs = tf.keras.Input(shape=(64, 64, 3))\nx = tf.keras.layers.experimental.preprocessing.Resizing(224, 224)(inputs)\n\n# Conv run in inference mode\nx = efficientnetb0_conv(x, training=False)\n\n# Rebuild the top\nx = tf.keras.layers.GlobalAveragePooling2D()(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.Dropout(0.2)(x)\noutputs = tf.keras.layers.Dense(10, activation='softmax')(x)\n\n# Full model\nnoisy_student_efficientnetb0 = tf.keras.Model(inputs, outputs)\n\nnoisy_student_efficientnetb0.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:00:13.740619Z","iopub.execute_input":"2021-09-25T12:00:13.741182Z","iopub.status.idle":"2021-09-25T12:00:16.319853Z","shell.execute_reply.started":"2021-09-25T12:00:13.741143Z","shell.execute_reply":"2021-09-25T12:00:16.319037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transfer learning\noptimizer = tf.keras.optimizers.Adam(learning_rate=0.01)\nnoisy_student_efficientnetb0.compile(\n    optimizer=optimizer, \n    loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.3), \n    metrics=[\"accuracy\"])\nearly_stop = tf.keras.callbacks.EarlyStopping(patience=3, verbose=1)\n\nhist1 = noisy_student_efficientnetb0.fit(train_ds, \n                                 validation_data=val_ds, \n                                 epochs=20, \n                                 batch_size=32, \n                                 verbose=1,\n                                 callbacks=[early_stop])\nplot(hist1)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:00:16.322045Z","iopub.execute_input":"2021-09-25T12:00:16.322387Z","iopub.status.idle":"2021-09-25T12:02:49.801436Z","shell.execute_reply.started":"2021-09-25T12:00:16.322351Z","shell.execute_reply":"2021-09-25T12:02:49.800695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fine tune the model\n\n# Unfreeze the efficientNetB0 convolution layers\n# However, the layers will still be runing in inference mode, effectively freezing BatchNorm layers while updating other layers\nefficientnetb0_conv.trainable = True\n\noptimizer = tf.keras.optimizers.Adam(learning_rate=0.00001)\nnoisy_student_efficientnetb0.compile(\n    optimizer=optimizer, \n    loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2), \n    metrics=[\"accuracy\"])\n\nhist2 = noisy_student_efficientnetb0.fit(train_ds, \n                                 validation_data=val_ds, \n                                 epochs=20, \n                                 batch_size=32, \n                                 verbose=1,\n                                 callbacks=[early_stop])\n\nplot(hist2)\n\nnoisy_student_efficientnetb0.save(\"noisy_student_efficientnetB0\")","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:02:49.804447Z","iopub.execute_input":"2021-09-25T12:02:49.804716Z","iopub.status.idle":"2021-09-25T12:13:16.984084Z","shell.execute_reply.started":"2021-09-25T12:02:49.804688Z","shell.execute_reply":"2021-09-25T12:13:16.983182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = noisy_student_efficientnetb0.predict(test_ds, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:18:01.522291Z","iopub.execute_input":"2021-09-25T12:18:01.522645Z","iopub.status.idle":"2021-09-25T12:18:24.951338Z","shell.execute_reply.started":"2021-09-25T12:18:01.522613Z","shell.execute_reply":"2021-09-25T12:18:24.950567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_predicted_labels = np.argmax(pred, axis=1)\nprint(num_predicted_labels[0:15])","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:18:24.953438Z","iopub.execute_input":"2021-09-25T12:18:24.954003Z","iopub.status.idle":"2021-09-25T12:18:24.960024Z","shell.execute_reply.started":"2021-09-25T12:18:24.953959Z","shell.execute_reply":"2021-09-25T12:18:24.959058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_labels = ['butterfly', 'cat', 'chicken', 'cow', 'dog', 'elephant', 'fish', 'horse', 'lion', 'spider']\ncat_predicted_labels= [categorical_labels[num_predicted_labels[i]] for i in range(len(num_predicted_labels))]\nprint(cat_predicted_labels[0:15])\nprint(Counter(cat_predicted_labels).values())","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:18:24.962245Z","iopub.execute_input":"2021-09-25T12:18:24.962689Z","iopub.status.idle":"2021-09-25T12:18:24.981673Z","shell.execute_reply.started":"2021-09-25T12:18:24.962580Z","shell.execute_reply":"2021-09-25T12:18:24.980481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_solution():\n    import csv\n    header= ['ID', 'Label']\n    with open('asdasd.csv', 'w', encoding='UTF8', newline='') as f:\n        writer = csv.writer(f)\n        # write the header\n        writer.writerow(header)\n        data = []\n        for i in range(len(cat_predicted_labels)):\n            data.append([str(i), cat_predicted_labels[i]])\n        writer.writerows(data)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:18:24.983124Z","iopub.execute_input":"2021-09-25T12:18:24.983484Z","iopub.status.idle":"2021-09-25T12:18:24.991018Z","shell.execute_reply.started":"2021-09-25T12:18:24.983449Z","shell.execute_reply":"2021-09-25T12:18:24.990037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_solution()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T12:18:24.992172Z","iopub.execute_input":"2021-09-25T12:18:24.992862Z","iopub.status.idle":"2021-09-25T12:18:25.034772Z","shell.execute_reply.started":"2021-09-25T12:18:24.992824Z","shell.execute_reply":"2021-09-25T12:18:25.034022Z"},"trusted":true},"execution_count":null,"outputs":[]}]}