{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is to practice trying coding a CNN from scratch so that I understand each bell and whistle of a CNN network and what could possibly go wrong when coding it from scratch.  <br> <br>\n\nIn this notebook, we will use:\n1. One hard coded VGG architecture without transfer learning <br>\n2. Small NN <br>\n3. VGG-16 transfer learning <br> <br>\n\nWithout transfer learning, the result is simply terrible. The training set is too small for the VGG architecture to detect anything. <br>\nA smaller network perform way better than VGG\n\n## Accuracy\n**VGG (no pretrained) - 0.5000 <br>\nSmall NN - 0.7681 <br>\nVGG 16 (Transfer Learning) - 0.9056 <br>**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.preprocessing.image import ImageDataGenerator, load_img\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image\nfrom tensorflow.keras.optimizers import RMSprop\nfrom keras.applications import VGG16\nfrom keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense, Activation,GlobalMaxPooling2D\nfrom keras.models import Model\nfrom keras import optimizers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/train.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"train\")\n\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/test1.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"test1\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_directory = \"train/train/\"\ntest_directory  = \"test1/test1/\"\n\n# See sample image\nfilenames = os.listdir(train_directory)\nsample = random.choice(filenames)\nprint(sample)\nimage = load_img(train_directory + sample)\nplt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 8000 train samples\n# 1600 validation samples\nimport shutil\n\nsource_dir = 'train/'\ndef copy_files(prefix_str, range_start, range_end, target_dir):\n    image_paths = []\n    for i in range(range_start, range_end):\n        image_path = os.path.join(source_dir,'train', prefix_str + '.'+ str(i)+ '.jpg')\n        image_paths.append(image_path)\n    dest_dir = os.path.join( 'data', target_dir, prefix_str)\n    os.makedirs(dest_dir)\n\n    for image_path in image_paths:\n        shutil.copy(image_path,  dest_dir)\n\ncopy_files('dog', 0, 4000, 'train')\ncopy_files('cat', 0, 4000, 'train')\ncopy_files('dog', 4000, 4800,'validation')\ncopy_files('cat', 4000, 4800, 'validation')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(os.listdir('data/train/cat')))\nprint(len(os.listdir('data/train/dog')))\nprint(len(os.listdir('data/validation/cat')))\nprint(len(os.listdir('data/validation/dog')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#remove train folder to free up space\nif  os.path.exists('train'):\n    #os.removedirs(\"train\")\n    shutil.rmtree(\"train\") ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* ## Building the VGG architecture: \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# adding the VGG architecture below\nImage(\"../input/vgg-architecture-image/VGG.png\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model 1: VGG (Not Pre-trained)"},{"metadata":{"trusted":true},"cell_type":"code","source":"## p.s.: VGG model is too big for kaggle to run (out of memory), removed a few FC layers and reduced the one dense layer size from 4096 to 1024\n\nmodel_vgg = tf.keras.models.Sequential([\n           tf.keras.layers.Conv2D(filters = 64, kernel_size = (3,3), padding = 'same', activation = 'relu', input_shape=(128,128,3)),\n           tf.keras.layers.Conv2D(filters = 64, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.MaxPooling2D(pool_size=2, strides=None, padding='valid'),\n           tf.keras.layers.Conv2D(filters = 128, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.Conv2D(filters = 128, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.MaxPooling2D(pool_size=2, strides=None, padding='valid'),\n           tf.keras.layers.Conv2D(filters = 256, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.Conv2D(filters = 256, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.Conv2D(filters = 256, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.MaxPooling2D(pool_size=2, strides=None, padding='valid'),\n           tf.keras.layers.Conv2D(filters = 512, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.Conv2D(filters = 512, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.Conv2D(filters = 512, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.MaxPooling2D(pool_size=2, strides=None, padding='valid'),\n           tf.keras.layers.Conv2D(filters = 512, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.Conv2D(filters = 512, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.Conv2D(filters = 512, kernel_size = (3,3), padding = 'same', activation = 'relu'),\n           tf.keras.layers.MaxPooling2D(pool_size=2, strides=None, padding='valid'),\n           tf.keras.layers.Flatten(),\n           tf.keras.layers.Dense(1024, activation='relu'),\n#            tf.keras.layers.Dense(4096, activation='relu'),\n#            tf.keras.layers.Dense(4096, activation='relu'),\n           tf.keras.layers.Dense(1, activation='sigmoid')\n])\n\nmodel_vgg.compile(optimizer=RMSprop(lr=0.001), loss='binary_crossentropy', metrics=['acc'])\n\nmodel_vgg.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_dir = 'data/train'\nvalidation_data_dir = 'data/validation'\n\ntrain_datagen = ImageDataGenerator(rescale=1./255,\n      rotation_range=40,\n      width_shift_range=0.2,\n      height_shift_range=0.2,\n      shear_range=0.2,\n      zoom_range=0.2,\n      horizontal_flip=True,\n      fill_mode='nearest')\ntrain_generator = train_datagen.flow_from_directory(train_data_dir,\n                                                    batch_size=32,\n                                                    class_mode='binary',\n                                                    target_size=(128,128))\n\nvalidation_datagen = ImageDataGenerator(rescale=1./255,\n      rotation_range=40,\n      width_shift_range=0.2,\n      height_shift_range=0.2,\n      shear_range=0.2,\n      zoom_range=0.2,\n      horizontal_flip=True,\n      fill_mode='nearest')\nvalidation_generator = validation_datagen.flow_from_directory(validation_data_dir,\n                                                              batch_size=32,\n                                                              class_mode='binary',\n                                                              target_size=(128,128))\n\nhistory = model_vgg.fit_generator(train_generator,\n                              epochs=15,\n                              verbose=1,\n                              validation_data=validation_generator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport matplotlib.image  as mpimg\nimport matplotlib.pyplot as plt\n\n#-----------------------------------------------------------\n# Retrieve a list of list results on training and test data\n# sets for each training epoch\n#-----------------------------------------------------------\nacc=history.history['acc']\nval_acc=history.history['val_acc']\nloss=history.history['loss']\nval_loss=history.history['val_loss']\n\nepochs=range(len(acc)) # Get number of epochs\n\n#------------------------------------------------\n# Plot training and validation accuracy per epoch\n#------------------------------------------------\nplt.plot(epochs, acc, 'r', \"Training Accuracy\")\nplt.plot(epochs, val_acc, 'b', \"Validation Accuracy\")\nplt.title('Training and validation accuracy')\nplt.figure()\n\n#------------------------------------------------\n# Plot training and validation loss per epoch\n#------------------------------------------------\nplt.plot(epochs, loss, 'r', \"Training Loss\")\nplt.plot(epochs, val_loss, 'b', \"Validation Loss\")\nplt.figure()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model 2: Small NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_nn = tf.keras.models.Sequential([\n    tf.keras.layers.Conv2D(16, (3, 3), activation='relu', input_shape=(128, 128, 3)),\n    tf.keras.layers.MaxPooling2D(2, 2),\n    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),\n    tf.keras.layers.MaxPooling2D(2, 2),\n    tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n    tf.keras.layers.MaxPooling2D(2, 2),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(512, activation='relu'),\n    tf.keras.layers.Dense(1, activation='sigmoid')\n])\n\nmodel_nn.compile(optimizer=RMSprop(lr=0.001), loss='binary_crossentropy', metrics=['acc'])\n\nhistory = model_nn.fit_generator(train_generator,\n                              epochs=15,\n                              verbose=1,\n                              validation_data=validation_generator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#-----------------------------------------------------------\n# Retrieve a list of list results on training and test data\n# sets for each training epoch\n#-----------------------------------------------------------\nacc=history.history['acc']\nval_acc=history.history['val_acc']\nloss=history.history['loss']\nval_loss=history.history['val_loss']\n\nepochs=range(len(acc)) # Get number of epochs\n\n#------------------------------------------------\n# Plot training and validation accuracy per epoch-++2-\n#------------------------------------------------\nplt.plot(epochs, acc, 'r', \"Training Accuracy\")\nplt.plot(epochs, val_acc, 'b', \"Validation Accuracy\")\nplt.title('Training and validation accuracy')\nplt.figure()\n\n#------------------------------------------------\n# Plot training and validation loss per epoch\n#------------------------------------------------\nplt.plot(epochs, loss, 'r', \"Training Loss\")\nplt.plot(epochs, val_loss, 'b', \"Validation Loss\")\nplt.figure()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model 3: VGG (Transfer Learning)"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_vgg_pretrained = VGG16(input_shape=(128, 128, 3), include_top=False, weights=\"imagenet\")\nmodel_vgg_pretrained.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#freeze all layers\nfor layer in model_vgg_pretrained.layers[:15]:\n    layer.trainable = False\n\nfor layer in model_vgg_pretrained.layers[15:]:\n    layer.trainable = True\n    \nlast_layer = model_vgg_pretrained.get_layer('block5_pool')\nlast_output = last_layer.output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Flatten the output layer to 1 dimension\nx = GlobalMaxPooling2D()(last_output)\n# Add a fully connected layer with 512 hidden units and ReLU activation\nx = Dense(512, activation='relu')(x)\n# Add a dropout rate of 0.5\nx = Dropout(0.5)(x)\n# Add a final sigmoid layer for classification\nx = Dense(1, activation='sigmoid')(x)\n\nmodel_vgg_pretrained = Model(model_vgg_pretrained.input, x)\n\nmodel_vgg_pretrained.compile(loss='binary_crossentropy',\n              optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),\n              metrics=['accuracy'])\n\nmodel_vgg_pretrained.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_vgg_pretrained.compile(optimizer=RMSprop(lr=0.001), loss='binary_crossentropy', metrics=['acc'])\n\nhistory = model_vgg_pretrained.fit_generator(train_generator,\n                              epochs=15,\n                              verbose=1,\n                              validation_data=validation_generator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#-----------------------------------------------------------\n# Retrieve a list of list results on training and test data\n# sets for each training epoch\n#-----------------------------------------------------------\nacc=history.history['acc']\nval_acc=history.history['val_acc']\nloss=history.history['loss']\nval_loss=history.history['val_loss']\n\nepochs=range(len(acc)) # Get number of epochs\n\n#------------------------------------------------\n# Plot training and validation accuracy per epoch-++2-\n#------------------------------------------------\nplt.plot(epochs, acc, 'r', \"Training Accuracy\")\nplt.plot(epochs, val_acc, 'b', \"Validation Accuracy\")\nplt.title('Training and validation accuracy')\nplt.figure()\n\n#------------------------------------------------\n# Plot training and validation loss per epoch\n#------------------------------------------------\nplt.plot(epochs, loss, 'r', \"Training Loss\")\nplt.plot(epochs, val_loss, 'b', \"Validation Loss\")\nplt.figure()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating test data and predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filenames = os.listdir(\"test1/test1\")\ntest_df = pd.DataFrame({\n    'filename': test_filenames\n})\nnb_samples = test_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_gen = ImageDataGenerator(rescale=1./255)\ntest_generator = test_gen.flow_from_dataframe(\n    test_df, \n    \"test1/test1\", \n    batch_size=32,\n    class_mode=None,\n    target_size=(128,128)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict = model_vgg_pretrained.predict_generator(test_generator, steps=np.ceil(nb_samples/32))\nthreshold = 0.5\ntest_df['category'] = np.where(predict > threshold, 1,0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_test = test_df.sample(n=9).reset_index()\nsample_test.head()\nplt.figure(figsize=(12, 12))\nfor index, row in sample_test.iterrows():\n    filename = row['filename']\n    category = row['category']\n    img = load_img(\"test1/test1/\"+filename, target_size=(256, 256))\n    plt.subplot(3, 3, index+1)\n    plt.imshow(img)\n    plt.xlabel(filename + '(' + \"{}\".format(category) + ')')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nsubmission_df = test_df.copy()\nsubmission_df['id'] = submission_df['filename'].str.split('.').str[0]\nsubmission_df['label'] = submission_df['category']\nsubmission_df.drop(['filename', 'category'], axis=1, inplace=True)\nsubmission_df.to_csv('submission_20200202.csv', index=False)\n\nplt.figure(figsize=(10,5))\nsns.countplot(submission_df['label'])\nplt.title(\"(Test data)\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}