{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import keras\nkeras.__version__","metadata":{"id":"9X6HNDCbPD7P","scrolled":true,"outputId":"495719b0-6d5c-4d02-a4d2-5c3b42018991","execution":{"iopub.status.busy":"2021-07-17T10:34:07.157749Z","iopub.execute_input":"2021-07-17T10:34:07.158102Z","iopub.status.idle":"2021-07-17T10:34:11.655301Z","shell.execute_reply.started":"2021-07-17T10:34:07.158071Z","shell.execute_reply":"2021-07-17T10:34:11.654461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nprint(os.listdir(\"../input/dogs-vs-cats\"))\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:34:11.656943Z","iopub.execute_input":"2021-07-17T10:34:11.657344Z","iopub.status.idle":"2021-07-17T10:34:11.663478Z","shell.execute_reply.started":"2021-07-17T10:34:11.657304Z","shell.execute_reply":"2021-07-17T10:34:11.662436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Split data**","metadata":{}},{"cell_type":"code","source":"import zipfile\n\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/train.zip\",\"r\") as z:\n    z.extractall(\".\")\n\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/test1.zip\",\"r\") as z:\n    z.extractall(\".\")\n\n!ls ./","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:34:11.665882Z","iopub.execute_input":"2021-07-17T10:34:11.666279Z","iopub.status.idle":"2021-07-17T10:34:28.785789Z","shell.execute_reply.started":"2021-07-17T10:34:11.666240Z","shell.execute_reply":"2021-07-17T10:34:28.784810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_dataset_dir = './train'\nbase_dir = '../data'\nif not os.path.isdir(base_dir): os.mkdir(base_dir)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:34:28.789467Z","iopub.execute_input":"2021-07-17T10:34:28.789734Z","iopub.status.idle":"2021-07-17T10:34:28.796609Z","shell.execute_reply.started":"2021-07-17T10:34:28.789704Z","shell.execute_reply":"2021-07-17T10:34:28.795800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = os.path.join(base_dir,'train')\nif not os.path.isdir(train_dir): os.mkdir(train_dir)\ntest_dir = os.path.join(base_dir,'test')\nif not os.path.isdir(test_dir): os.mkdir(test_dir)\nvalidation_dir = os.path.join(base_dir,'validation')\nif not os.path.isdir(validation_dir): os.mkdir(validation_dir)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:34:28.800021Z","iopub.execute_input":"2021-07-17T10:34:28.800314Z","iopub.status.idle":"2021-07-17T10:34:28.808820Z","shell.execute_reply.started":"2021-07-17T10:34:28.800279Z","shell.execute_reply":"2021-07-17T10:34:28.807909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cats_dir = os.path.join(train_dir,'cats')\nif not os.path.isdir(train_cats_dir): os.mkdir(train_cats_dir)\n\ntrain_dogs_dir = os.path.join(train_dir,'dogs')\nif not os.path.isdir(train_dogs_dir): os.mkdir(train_dogs_dir)\n    \nvalidation_cats_dir = os.path.join(validation_dir,'cats')\nif not os.path.isdir(validation_cats_dir) : os.mkdir(validation_cats_dir)\nvalidation_dogs_dir = os.path.join(validation_dir,'dogs')\nif not os.path.isdir(validation_dogs_dir) : os.mkdir(validation_dogs_dir)\n\ntest_cats_dir = os.path.join(test_dir,'cats')\nif not os.path.isdir(test_cats_dir) : os.mkdir(test_cats_dir)\ntest_dogs_dir = os.path.join(test_dir,'dogs')\nif not os.path.isdir(test_dogs_dir) : os.mkdir(test_dogs_dir)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:34:28.810231Z","iopub.execute_input":"2021-07-17T10:34:28.810575Z","iopub.status.idle":"2021-07-17T10:34:28.820718Z","shell.execute_reply.started":"2021-07-17T10:34:28.810539Z","shell.execute_reply":"2021-07-17T10:34:28.819684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def showdir(path, depth):\n    if depth == 0:\n        print(\"root:[\" + path + \"]\")\n \n    for item in os.listdir(path):\n        if '.git' not in item:\n            print(\"|      \" * depth + \"|--\" + item)\n \n            newitem = os.path.join(path,item)\n            if os.path.isdir(newitem):\n                showdir(newitem, depth +1)\nshowdir(base_dir,0)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:34:28.822386Z","iopub.execute_input":"2021-07-17T10:34:28.822875Z","iopub.status.idle":"2021-07-17T10:34:28.837517Z","shell.execute_reply.started":"2021-07-17T10:34:28.822837Z","shell.execute_reply":"2021-07-17T10:34:28.836676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nfnames = ['cat.{}.jpg'.format(i) for i in range(1000)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(train_cats_dir,fname)\n    shutil.copyfile(src,dst)   \nfnames = ['cat.{}.jpg'.format(i) for i in range(1000,1500)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(validation_cats_dir,fname)\n    shutil.copyfile(src,dst)\nfnames = ['cat.{}.jpg'.format(i) for i in range(1500,2000)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(test_cats_dir,fname)\n    shutil.copyfile(src,dst)\n\nfnames = ['dog.{}.jpg'.format(i) for i in range(1000)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(train_dogs_dir,fname)\n    shutil.copyfile(src,dst)   \nfnames = ['dog.{}.jpg'.format(i) for i in range(1000,1500)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(validation_dogs_dir,fname)\n    shutil.copyfile(src,dst)\nfnames = ['dog.{}.jpg'.format(i) for i in range(1500,2000)]\nfor fname in fnames:\n    src = os.path.join(original_dataset_dir, fname)\n    dst = os.path.join(test_dogs_dir,fname)\n    shutil.copyfile(src,dst)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:34:28.841750Z","iopub.execute_input":"2021-07-17T10:34:28.842002Z","iopub.status.idle":"2021-07-17T10:34:29.242038Z","shell.execute_reply.started":"2021-07-17T10:34:28.841978Z","shell.execute_reply":"2021-07-17T10:34:29.241185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ncats = pd.Series([len(os.listdir(train_cats_dir)),len(os.listdir(validation_cats_dir)),len(os.listdir(test_cats_dir))])\ndogs = pd.Series([len(os.listdir(train_dogs_dir)),len(os.listdir(validation_dogs_dir)),len(os.listdir(test_dogs_dir))])\ndf = pd.DataFrame({'Cats':cats,'Dogs':dogs})\ndf.index = ['Train','Validation','Test']\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:34:29.243827Z","iopub.execute_input":"2021-07-17T10:34:29.244167Z","iopub.status.idle":"2021-07-17T10:34:29.278168Z","shell.execute_reply.started":"2021-07-17T10:34:29.244141Z","shell.execute_reply":"2021-07-17T10:34:29.277205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **VGG16**","metadata":{}},{"cell_type":"markdown","source":"I use a convolutional base layer of a VGG16 network trained on the ImageNet dataset to extract useful features from dog and cat images. Then I'll train the Dogs vs. Cats classifier with this features.\n\nLet's create a VGG16 model:","metadata":{"id":"pnWzdEQFPD7k"}},{"cell_type":"code","source":"import tensorflow as tf\nimport keras\n\nfrom tensorflow.keras.applications.vgg16 import VGG16\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications.vgg16 import preprocess_input\nimport numpy as np\n\nconv_base = VGG16(weights='imagenet',\n                  include_top=False,\n                  input_shape=(150, 150, 3))","metadata":{"id":"JAapyiZhPD7l","execution":{"iopub.status.busy":"2021-07-17T10:34:29.279607Z","iopub.execute_input":"2021-07-17T10:34:29.279959Z","iopub.status.idle":"2021-07-17T10:34:32.189188Z","shell.execute_reply.started":"2021-07-17T10:34:29.279924Z","shell.execute_reply":"2021-07-17T10:34:32.188227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The VGG16 function takes three parameters: `weights`, `include_top`, `input_shape`\n\nBelow is the detailed structure of the VGG16 convolutional base layer.","metadata":{"id":"FPQ4wkkfPD7m"}},{"cell_type":"code","source":"conv_base.summary()","metadata":{"id":"JTiTr0qMPD7o","outputId":"70aba23e-95d3-4431-d330-98c6cc37161f","execution":{"iopub.status.busy":"2021-07-17T10:34:32.190470Z","iopub.execute_input":"2021-07-17T10:34:32.190797Z","iopub.status.idle":"2021-07-17T10:34:32.206079Z","shell.execute_reply.started":"2021-07-17T10:34:32.190763Z","shell.execute_reply":"2021-07-17T10:34:32.205101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The size of the final feature map is `(4, 4, 512)`. On top of this feature, I will put a fully connected layer. Two methods are available at this point.\n\n1. Run the convolutional base layer on the new dataset and save the output to disk as a NumPy array. Use this data as input to an independent fully connected classifier.\n\n2. Build a Dense layer on top of the prepared model(In this notebook, `conv_base`) and expand it. It then runs the entire model end-to-end on the input data.\n\nI will try both methods.","metadata":{"id":"8g1Yb7Q1PD7x"}},{"cell_type":"markdown","source":"# **1. Feature extract without data augmentation (fast, low cost, risk of overfitting)**","metadata":{"id":"3Fdu8wzFLfFQ"}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ntrain_dir = os.path.join(base_dir, 'train')\nvalidation_dir = os.path.join(base_dir, 'validation')\ntest_dir = os.path.join(base_dir, 'test')\n\ndatagen = ImageDataGenerator(rescale=1./255)\nbatch_size = 20\n\ndef extract_features(directory, sample_count):\n    features = np.zeros(shape=(sample_count, 4, 4, 512))\n    labels = np.zeros(shape=(sample_count))\n    generator = datagen.flow_from_directory(\n        directory,\n        target_size=(150, 150),\n        batch_size=batch_size,\n        class_mode='binary')\n    i = 0\n    for inputs_batch, labels_batch in generator:\n        features_batch = conv_base.predict(inputs_batch)\n        features[i * batch_size : (i + 1) * batch_size] = features_batch\n        labels[i * batch_size : (i + 1) * batch_size] = labels_batch\n        i += 1\n        if i * batch_size >= sample_count:\n            break\n    return features, labels\n\ntrain_features, train_labels = extract_features(train_dir, 2000)\nvalidation_features, validation_labels = extract_features(validation_dir, 1000)\ntest_features, test_labels = extract_features(test_dir, 1000)","metadata":{"id":"XqaCcV4gPD7y","outputId":"9d4f61d0-29e5-424e-8486-aeb6ec7b2aa8","execution":{"iopub.status.busy":"2021-07-17T10:34:32.207440Z","iopub.execute_input":"2021-07-17T10:34:32.207939Z","iopub.status.idle":"2021-07-17T10:35:04.670591Z","shell.execute_reply.started":"2021-07-17T10:34:32.207900Z","shell.execute_reply":"2021-07-17T10:35:04.669636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The size of the extracted features is `(samples, 4, 4, 512)`. To put it in the fully connected classifier, expand it to size `(samples, 8192)`:","metadata":{"id":"A8P41817PD71"}},{"cell_type":"code","source":"train_features = np.reshape(train_features, (2000, 4 * 4 * 512))\nvalidation_features = np.reshape(validation_features, (1000, 4 * 4 * 512))\ntest_features = np.reshape(test_features, (1000, 4 * 4 * 512))","metadata":{"id":"IjjLt5j9PD72","execution":{"iopub.status.busy":"2021-07-17T10:35:04.672045Z","iopub.execute_input":"2021-07-17T10:35:04.672403Z","iopub.status.idle":"2021-07-17T10:35:04.680388Z","shell.execute_reply.started":"2021-07-17T10:35:04.672360Z","shell.execute_reply":"2021-07-17T10:35:04.677472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then I define a fully connected classifier and train it using the stored data and labels:","metadata":{"id":"7nxvYER-PD73"}},{"cell_type":"code","source":"from keras import models\nfrom keras import layers\nfrom keras import optimizers\n\nmodel = models.Sequential()\nmodel.add(layers.Dense(256, activation='relu', input_dim=4 * 4 * 512))\nmodel.add(layers.Dropout(0.5))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n\nmodel.compile(optimizer=optimizers.RMSprop(lr=2e-5),\n              loss='binary_crossentropy',\n              metrics=['acc'])\n\nhistory = model.fit(train_features, train_labels,\n                    epochs=30,\n                    batch_size=20,\n                    validation_data=(validation_features, validation_labels))","metadata":{"id":"uba81fAwPD75","outputId":"b675d0e0-3385-4eb1-f418-159b33c42f44","execution":{"iopub.status.busy":"2021-07-17T10:35:04.681993Z","iopub.execute_input":"2021-07-17T10:35:04.682374Z","iopub.status.idle":"2021-07-17T10:35:16.892707Z","shell.execute_reply.started":"2021-07-17T10:35:04.682320Z","shell.execute_reply":"2021-07-17T10:35:16.891770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nacc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Training acc')\nplt.plot(epochs, val_acc, 'b', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Training loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"id":"lSSipIgmPD78","outputId":"ee74b5ac-aa55-42f4-fe37-41461e23b491","execution":{"iopub.status.busy":"2021-07-17T10:35:16.894617Z","iopub.execute_input":"2021-07-17T10:35:16.895003Z","iopub.status.idle":"2021-07-17T10:35:17.225317Z","shell.execute_reply.started":"2021-07-17T10:35:16.894962Z","shell.execute_reply":"2021-07-17T10:35:17.224444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A verification accuracy of 90% was reached. But even with a lot of dropout, it overfitted almost immediately as soon as it started training. This is because I did not use data augmentation, which is essential to avoid overfitting on small image datasets.","metadata":{"id":"I2Au2WHKPD7-"}},{"cell_type":"markdown","source":"# **2. Feature extract using data augmentation (slow, high cost, less overfitting)**","metadata":{"id":"KPkRQ4NWMRa7"}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nmodel = keras.Sequential()\nmodel.add(keras.Input(shape=(150,150,3)))\nmodel.add(conv_base)\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(256, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))","metadata":{"id":"TqrmiqbxPnSN","outputId":"58469dc5-5a53-405b-dfe7-f6f76ceb9121","execution":{"iopub.status.busy":"2021-07-17T10:35:17.226662Z","iopub.execute_input":"2021-07-17T10:35:17.227036Z","iopub.status.idle":"2021-07-17T10:35:17.299486Z","shell.execute_reply.started":"2021-07-17T10:35:17.226985Z","shell.execute_reply":"2021-07-17T10:35:17.298686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extend the `conv_base` model I made in step 1. Models work just like layers, so I can add other models to Sequential model like layers.\n\nThe structure of this model is this:","metadata":{"id":"rpP30CNxMf8_"}},{"cell_type":"code","source":"model.summary()","metadata":{"id":"TU0pQjQ7PD8A","outputId":"53975521-f11c-4d9e-d080-bf43d05c133a","execution":{"iopub.status.busy":"2021-07-17T10:35:17.300726Z","iopub.execute_input":"2021-07-17T10:35:17.301219Z","iopub.status.idle":"2021-07-17T10:35:17.311350Z","shell.execute_reply.started":"2021-07-17T10:35:17.301174Z","shell.execute_reply":"2021-07-17T10:35:17.310337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The convolution-based layer of VGG16 has 14714688 parameters, while the classifier added on top of the convolution-based layer has 2097408 parameters.\n    \n\n    \nSince I will be using a pre-trained model, the weights should not be updated in the training from now on. So I need to freeze the Convolutional Layer before compiling and training the model.   \n\nA frozen layer is not updated with weights during training. If you don't freeze here, it's useless to use a pre-trained model.","metadata":{"id":"YRH_w-lqPD8B"}},{"cell_type":"code","source":"conv_base.trainable = False","metadata":{"id":"n5r0YYhZPD8D","execution":{"iopub.status.busy":"2021-07-17T10:35:17.312936Z","iopub.execute_input":"2021-07-17T10:35:17.313317Z","iopub.status.idle":"2021-07-17T10:35:17.318402Z","shell.execute_reply.started":"2021-07-17T10:35:17.313281Z","shell.execute_reply":"2021-07-17T10:35:17.317231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I can start training the model using the data augmentation:","metadata":{"id":"puU5UTEZPD8F"}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(\n      rescale=1./255,\n      rotation_range=20,\n      width_shift_range=0.1,\n      height_shift_range=0.1,\n      shear_range=0.1,\n      zoom_range=0.1,\n      horizontal_flip=True,\n      fill_mode='nearest')\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntrain_generator = train_datagen.flow_from_directory(\n        train_dir,\n        target_size=(150, 150),\n        batch_size=20,\n        class_mode='binary')\n\nvalidation_generator = test_datagen.flow_from_directory(\n        validation_dir,\n        target_size=(150, 150),\n        batch_size=20,\n        class_mode='binary')\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer=optimizers.RMSprop(lr=2e-5),\n              metrics=['acc'])\n\nhistory = model.fit_generator(\n      train_generator,\n      steps_per_epoch=100,\n      epochs=30,\n      validation_data=validation_generator,\n      validation_steps=50,\n      verbose=2)","metadata":{"id":"C4rzr3yFPD8H","outputId":"34b6c22e-771a-4667-a557-e0b2446c8b2b","execution":{"iopub.status.busy":"2021-07-17T10:35:17.320040Z","iopub.execute_input":"2021-07-17T10:35:17.320401Z","iopub.status.idle":"2021-07-17T10:44:32.804073Z","shell.execute_reply.started":"2021-07-17T10:35:17.320365Z","shell.execute_reply":"2021-07-17T10:44:32.803257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(\n      train_generator,\n      steps_per_epoch=100,\n      epochs=30,\n      validation_data=validation_generator,\n      validation_steps=50,\n      verbose=2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = image.load_img('./train/cat.1665.jpg', target_size=(150,150))\nx=image.img_to_array(img)\nx=x.reshape((1,) + x.shape)\n\ni=0 \nfor batch in train_datagen.flow(x, batch_size=1):\n    plt.figure(i)\n    imgplot=plt.imshow(image.array_to_img(batch[0]))\n    i+=1\n    if i%6==0:\n        break\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-17T10:44:32.807143Z","iopub.execute_input":"2021-07-17T10:44:32.807399Z","iopub.status.idle":"2021-07-17T10:44:33.736388Z","shell.execute_reply.started":"2021-07-17T10:44:32.807373Z","shell.execute_reply":"2021-07-17T10:44:33.735399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Training acc')\nplt.plot(epochs, val_acc, 'b', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Training loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"id":"eaIg9jRPPD8L","outputId":"eac2c5a1-05eb-4c1a-b8af-b98e2c706d2b","execution":{"iopub.status.busy":"2021-07-17T10:44:33.737859Z","iopub.execute_input":"2021-07-17T10:44:33.738232Z","iopub.status.idle":"2021-07-17T10:44:34.308246Z","shell.execute_reply.started":"2021-07-17T10:44:33.738188Z","shell.execute_reply":"2021-07-17T10:44:34.307115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The verification accuracy is similar to the previous graph. However, the overfitting is slightly reduced.","metadata":{"id":"8mrYV-XNPD8M"}},{"cell_type":"markdown","source":"# **Fine tuning**\nFine-tuning is used to further adjusted to the problem given the reuse model.","metadata":{"id":"NtzemYAYOi_i"}},{"cell_type":"markdown","source":"**Network fine-tuning steps:**\n1. Add a new network on top of the pre-trained base network\n2. Freeze the underlying network\n3. Train the newly added network\n4. Unfreeze some layers in the underlying network\n5. Train the unfreeze layer and the newly added layer together","metadata":{"id":"XngrdoFBP42N"}},{"cell_type":"markdown","source":"Parts 1 to 3 have already been completed while extracting features. Here, I proceed with steps 4 and 5.","metadata":{"id":"NcL5HjRtP6_M"}},{"cell_type":"code","source":"conv_base.summary()","metadata":{"id":"pmIJ-q3oPD8R","outputId":"be5bbf7a-4c00-4a5b-e040-087a0ba20e1f","execution":{"iopub.status.busy":"2021-07-17T10:44:34.309826Z","iopub.execute_input":"2021-07-17T10:44:34.310236Z","iopub.status.idle":"2021-07-17T10:44:34.329490Z","shell.execute_reply.started":"2021-07-17T10:44:34.310185Z","shell.execute_reply":"2021-07-17T10:44:34.328624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's fine-tune the last three convolutional layers. In other words, all layers up to `block4_pool` are frozen, and `block5_conv1`, `block5_conv2`, and `block5_conv3` layers will be trained.","metadata":{"id":"tSUpEJEHPD8S"}},{"cell_type":"code","source":"conv_base.trainable = True\n\nset_trainable = False\nfor layer in conv_base.layers:\n    if layer.name == 'block5_conv1':\n        set_trainable = True\n    if set_trainable:\n        layer.trainable = True\n    else:\n        layer.trainable = False","metadata":{"id":"b295Co9xPD8T","execution":{"iopub.status.busy":"2021-07-17T10:44:34.333510Z","iopub.execute_input":"2021-07-17T10:44:34.333765Z","iopub.status.idle":"2021-07-17T10:44:34.341592Z","shell.execute_reply.started":"2021-07-17T10:44:34.333740Z","shell.execute_reply":"2021-07-17T10:44:34.340768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's start fine-tuning the network. Use the RMSProp optimizer with a lower learning rate.","metadata":{"id":"q-tXoOZHPD8V"}},{"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n              optimizer=optimizers.RMSprop(lr=1e-5),\n              metrics=['acc'])\n\nhistory = model.fit_generator(\n      train_generator,\n      steps_per_epoch=100,\n      epochs=100,\n      validation_data=validation_generator,\n      validation_steps=50)","metadata":{"id":"5_Y_L8a8PD8V","outputId":"6e9fc9da-0a0d-4968-83fa-77ac5a8493c8","execution":{"iopub.status.busy":"2021-07-17T10:44:34.343087Z","iopub.execute_input":"2021-07-17T10:44:34.343407Z","iopub.status.idle":"2021-07-17T11:16:41.874084Z","shell.execute_reply.started":"2021-07-17T10:44:34.343374Z","shell.execute_reply":"2021-07-17T11:16:41.873107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'bo', label='Training acc')\nplt.plot(epochs, val_acc, 'b', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo', label='Training loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"id":"H13JR_CXPD8g","outputId":"f0cef038-8a39-4b96-a448-41e3b70d1707","execution":{"iopub.status.busy":"2021-07-17T11:16:41.875909Z","iopub.execute_input":"2021-07-17T11:16:41.876418Z","iopub.status.idle":"2021-07-17T11:16:42.208753Z","shell.execute_reply.started":"2021-07-17T11:16:41.876370Z","shell.execute_reply":"2021-07-17T11:16:42.207605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The graph looks irregular. The graph can be expressed smoothly with the 'exponential moving average'.","metadata":{"id":"VhmKzfsPPD8g"}},{"cell_type":"code","source":"def smooth_curve(points, factor=0.8):\n  smoothed_points = []\n  for point in points:\n    if smoothed_points:\n      previous = smoothed_points[-1]\n      smoothed_points.append(previous * factor + point * (1 - factor))\n    else:\n      smoothed_points.append(point)\n  return smoothed_points\n\nplt.plot(epochs,\n         smooth_curve(acc), 'bo', label='Smoothed training acc')\nplt.plot(epochs,\n         smooth_curve(val_acc), 'b', label='Smoothed validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs,\n         smooth_curve(loss), 'bo', label='Smoothed training loss')\nplt.plot(epochs,\n         smooth_curve(val_loss), 'b', label='Smoothed validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","metadata":{"id":"52LE4gtXPD8h","outputId":"e3c05c33-a1d0-429e-939b-3ac001eb9ac9","execution":{"iopub.status.busy":"2021-07-17T11:19:45.344754Z","iopub.execute_input":"2021-07-17T11:19:45.345107Z","iopub.status.idle":"2021-07-17T11:19:45.633336Z","shell.execute_reply.started":"2021-07-17T11:19:45.345073Z","shell.execute_reply":"2021-07-17T11:19:45.632339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In terms of the accuracy of the graph, it has improved by about 1% compared to the previous graph. However, the loss curve has gotten worse. The reason that the loss can increase even though the accuracy is improved is that it is the distribution of the loss values that affects the accuracy, not the average.","metadata":{"id":"mKgzB-zLPD8i"}},{"cell_type":"markdown","source":"Finally, evaluate the model on the test data.","metadata":{"id":"4f7CvgBTRCq7"}},{"cell_type":"code","source":"test_generator = test_datagen.flow_from_directory(\n        test_dir,\n        target_size=(150, 150),\n        batch_size=20,\n        class_mode='binary')\n\ntest_loss, test_acc = model.evaluate_generator(test_generator, steps=50)\nprint('test acc:', test_acc)","metadata":{"id":"wP_BLV3UPD8j","outputId":"3b8d7306-3ec0-43c4-f06b-bf7418ccd8c3","execution":{"iopub.status.busy":"2021-07-17T11:16:42.536991Z","iopub.execute_input":"2021-07-17T11:16:42.537421Z","iopub.status.idle":"2021-07-17T11:16:45.861054Z","shell.execute_reply.started":"2021-07-17T11:16:42.537379Z","shell.execute_reply":"2021-07-17T11:16:45.860039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filenames = os.listdir('/kaggle/working/test1')\ntest_df = pd.DataFrame({'id': filenames})\ntest_size = test_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-07-17T11:26:58.383166Z","iopub.execute_input":"2021-07-17T11:26:58.383481Z","iopub.status.idle":"2021-07-17T11:26:58.399629Z","shell.execute_reply.started":"2021-07-17T11:26:58.383454Z","shell.execute_reply":"2021-07-17T11:26:58.398537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-17T11:26:59.793286Z","iopub.execute_input":"2021-07-17T11:26:59.793604Z","iopub.status.idle":"2021-07-17T11:26:59.799579Z","shell.execute_reply.started":"2021-07-17T11:26:59.793576Z","shell.execute_reply":"2021-07-17T11:26:59.798545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator_final = test_datagen.flow_from_dataframe(\n        test_df,\n        \"/kaggle/working/test1/\",\n        x_col=\"id\",\n        y_col=None,\n        target_size=(200, 200),\n        batch_size=20,\n        class_mode=None\n        )","metadata":{"execution":{"iopub.status.busy":"2021-07-17T11:27:13.470332Z","iopub.execute_input":"2021-07-17T11:27:13.470649Z","iopub.status.idle":"2021-07-17T11:27:13.588174Z","shell.execute_reply.started":"2021-07-17T11:27:13.470621Z","shell.execute_reply":"2021-07-17T11:27:13.587170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = model.predict_generator(test_generator_final, steps=50)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T11:28:07.171543Z","iopub.execute_input":"2021-07-17T11:28:07.171876Z","iopub.status.idle":"2021-07-17T11:28:10.145893Z","shell.execute_reply.started":"2021-07-17T11:28:07.171847Z","shell.execute_reply":"2021-07-17T11:28:10.144994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = test_df.copy()\nsubmission['id'] = submission['id'].str.split('.').str[0]\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T11:35:13.386683Z","iopub.execute_input":"2021-07-17T11:35:13.387071Z","iopub.status.idle":"2021-07-17T11:35:13.433540Z","shell.execute_reply.started":"2021-07-17T11:35:13.387032Z","shell.execute_reply":"2021-07-17T11:35:13.432682Z"},"trusted":true},"execution_count":null,"outputs":[]}]}