{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":5441,"databundleVersionId":38425}],"dockerImageVersionId":25160,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"dataset_subset = False # Train with only a subset of the data for quick tests\ndata_subset_size = 2000 # Subset size (for each class)\ncolor = True # Keep the color dimension or else load the data in greyscale\ndata_generation = True # Perform data augmentation\nassign_test_labels = False # Sets all test predictions to either 1 or 0\nquick_training = False # Reduces the number of epochs to a 10%\n\nearly_stop_overfitting = True; # Stop the training if the model doesn't improve in order to prevent overfitting\nlearning_rate_smoothing = True; # Reduces the learning rate of the backpropagation during the fitting if the model isn't improving\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Regular CNNs hiperparameters\nbatch_size = 16\nnum_clases = 1\nepochs = 100\nconv_kernel_size = 3\n\n# CNN fine tuning hiperparameters\ndefault_dropout_rate = 0.2\nregularizaion_weight = 0.001\nlearning_rate_reduction_factor = 0.5\n\n# Data hiperparameters\nimg_width = 132\nimg_height = 132\nvalidation_size = 0.2\n\nif color:\n    img_channels = 3\nelse:\n    img_channels = 1\n    \nif quick_training:\n    epochs = epochs * 0.1\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os # data fetching\nimport random # training set shuffling\nimport gc # garbage collector to clean memory\nimport cv2 # image preprocessing\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dataset directory check\nprint(os.listdir(\"../input/\"))\n","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dir = '../input/train'\ntest_dir = '../input/test'\n\nif dataset_subset:\n    train_dogs = ['../input/train/{}'.format(filename) for filename in os.listdir(train_dir) if 'dog' in filename]\n    train_cats = ['../input/train/{}'.format(filename) for filename in os.listdir(train_dir) if 'cat' in filename]\n    \n    # Only a small portion of both classes is used, in favor of quicker \n    train_imgs = train_dogs[:data_subset_size] + train_cats[:data_subset_size]\n    \n    # Memory freeing tasks\n    del train_dogs\n    del train_cats\n    gc.collect()\n    \nelse:\n    train_imgs = ['../input/train/{}'.format(filename) for filename in os.listdir(train_dir)]\n    \n\nrandom.shuffle(train_imgs)\n\ntest_imgs = ['../input/test/{}'.format(test_img) for test_img in os.listdir(test_dir)]\n# The ids gets processed as just the numbers in the filename as integers, without the extension\ntest_ids = [int(test_img[14:-4]) for test_img in test_imgs]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np # linear algebra\nfrom sklearn.model_selection import train_test_split # train-validation splitter","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_images(img_path_list):\n    \"\"\"\n    Loads and preprocesses all the images whose paths included in img_path_list\n    Return\n        X: array of resized images\n        y: array of labels\n    \"\"\"\n    X = []\n    y = []\n    \n    for img_path in img_path_list:\n        if color:\n            x = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)\n            x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)\n            # This last bit is to have the images coverted from the default BGR from cv2\n            # to RGB to correctly visualize the dataset (it has no effect over the training)\n        else:\n            x = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n            \n        x = cv2.resize(x, (img_height, img_width))\n        X.append(x)\n            \n        if 'dog' in img_path:\n            y.append(1)          \n        elif 'cat' in img_path:\n            y.append(0)\n\n    return X, y\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X, y = preprocess_images(train_imgs)\n\ndel train_imgs\ngc.collect()\n\n# Validation set splitting\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=validation_size)\n\ndel X\ndel y\ngc.collect()\n\nX_train = np.array(X_train)\nX_val = np.array(X_val)\ny_train = np.array(y_train)\ny_val = np.array(y_val)\n\ntrain_size = X_train.shape[0]\nval_size = X_val.shape[0]\n\nprint(\"Train and validation shapes\")\nprint(\"X_train: \" + str(X_train.shape))\nprint(\"X_val: \" + str(X_val.shape))\nprint(\"y_train: \" + str(y_train.shape))\nprint(\"y_val: \" + str(y_val.shape))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test, _ = preprocess_images(test_imgs)\n\nX_test = np.array(X_test).astype('float32')\n\n# If data generation is beign used, as it will transform train images to float (and, so, values \n# from 0 to 1), test dataset should be adapted to what our model is going to learn to treat.\nif data_generation:\n    X_test /= 255\n\n# Memory liberation tasks\ndel test_imgs\ngc.collect\n    \nprint(\"Test dataset shape: \")\nprint(X_test.shape)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator \n# Documentation: https://keras.io/preprocessing/image/\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if data_generation:\n    \n    # Fourth dimension addition in case of its value being onesized\n    if img_channels == 1:\n        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2], 1)\n        X_val = X_val.reshape(X_val.shape[0], X_val.shape[1], X_val.shape[2], 1)\n        \n    data_augmentator = ImageDataGenerator(rescale=1./255, rotation_range=0.2, shear_range=0.1, zoom_range=0.2,\n                                          width_shift_range=0.1, height_shift_range=0.1, fill_mode='reflect',horizontal_flip=True)\n    data_augmentator.fit(X_train)\n    data_generator = data_augmentator.flow(X_train, y_train, batch_size=batch_size)\n    \n    val_augmentator = ImageDataGenerator(rescale=1./255)\n    val_generator = val_augmentator.flow(X_val, y_val, batch_size=batch_size)\n\n    # Dimension restitution\n    if img_channels == 1:\n        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2])\n        X_val = X_val.reshape(X_val.shape[0], X_val.shape[1], X_val.shape[2])\n        ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The good, the bad and the ugly\n%matplotlib inline \n\nfrom matplotlib import pyplot as plt # data visualization\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_data(X, y, num_figures):\n    \"\"\"\n    Prints the images stored in X, with their correspondent labels in y.\n    num_figures images by row.\n    \"\"\"\n    plt.figure(figsize=(30, 20))\n\n    for i in range(num_figures):\n        plt.subplot(2, num_figures, i+1)\n        if color:\n            plt.imshow(X[i])\n        else:\n            plt.imshow(X[i], cmap='gray')\n        if y[i] >= 0.5:\n            plt.title(\"Doge (\"+ str(y[i]) + \")\", fontsize=30)\n        else:\n            plt.title(\"Catto (\"+ str(y[i]) + \")\", fontsize=30)\n            \n    plt.tight_layout()\n    plt.show()    \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Preprocess training dataset showcase with labels\nfor i in range(0, 24, 6): \n    plot_data(X_train[i:], y_train[i:], 6)\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if data_generation:\n    \n    for X_train_gen, y_train_gen in data_generator:\n        \n        if img_channels == 1:\n            X_train_gen = X_train_gen.reshape(X_train_gen.shape[0], X_train_gen.shape[1], X_train_gen.shape[2])\n        \n        print(\"X_train_gen shape: \" + str(X_train_gen.shape))\n        print(\"y_train_gen shape: \" + str(y_train_gen.shape))\n        \n        for i in range(0, batch_size-6, 6):\n            plot_data(X_train_gen[i:], y_train_gen[i:], 6)\n            \n        del X_train_gen\n        del y_train_gen\n        gc.collect()\n        \n        break\n\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Imports\nfrom tensorflow.keras.models import Sequential # Documentation: https://keras.io/models/sequential/\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense # Documentation: https://keras.io/layers/core/, https://keras.io/layers/convolutional/\nfrom tensorflow.keras.layers import Dropout, SpatialDropout2D, BatchNormalization\nfrom tensorflow.keras.optimizers import Adam, RMSprop # Documentation: https://keras.io/optimizers/\nfrom tensorflow.keras.regularizers import l2 # Documentation: https://keras.io/regularizers/\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_regularization_layer(model, type, rate=default_dropout_rate):\n    \"\"\"\n    Adds a regularization layer to the model based on the active control hiperparameters.\n    It's open to multipple addition, although you probably want to add only one of them.\n    \n    'rate' parameter only affects dropout layers.\n    \"\"\"\n    if type == \"batch_normalization\":\n        model.add(BatchNormalization())\n    if type == \"spatial_dropout\":\n        model.add(SpatialDropout2D(rate)) \n    if type == \"dropout\":\n        model.add(Dropout(rate))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32, kernel_size=(conv_kernel_size, conv_kernel_size), activation='relu', input_shape=(img_width, img_height, img_channels))) # Strides are, by default, (1,1)\nadd_regularization_layer(model,\"batch_normalization\")\nadd_regularization_layer(model,\"spatial_dropout\", 0.25)\nmodel.add(MaxPooling2D(pool_size=(2,2))) # Strides are, by default, of the same size of the pool size\n\n\nmodel.add(Conv2D(64, kernel_size=(conv_kernel_size, conv_kernel_size), activation='relu'))\nadd_regularization_layer(model,\"batch_normalization\")\nadd_regularization_layer(model,\"spatial_dropout\", 0.25)\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n\n\nmodel.add(Conv2D(128, kernel_size=(conv_kernel_size, conv_kernel_size), activation='relu'))\nadd_regularization_layer(model,\"batch_normalization\")\nadd_regularization_layer(model,\"spatial_dropout\", 0.25)\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n\nmodel.add(Conv2D(256, kernel_size=(conv_kernel_size, conv_kernel_size), activation='relu'))\nadd_regularization_layer(model,\"batch_normalization\")\nadd_regularization_layer(model,\"spatial_dropout\", 0.25)\nmodel.add(Conv2D(256, kernel_size=(conv_kernel_size, conv_kernel_size), activation='relu'))\nadd_regularization_layer(model,\"batch_normalization\")\nadd_regularization_layer(model,\"spatial_dropout\", 0.25)\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n\nmodel.add(Flatten()) \n\nmodel.add(Dense(1024, activation='relu', kernel_regularizer=l2(regularizaion_weight)))\nadd_regularization_layer(model,\"batch_normalization\")\nadd_regularization_layer(model,\"dropout\", 0.5)\nmodel.add(Dense(1024, activation='relu', kernel_regularizer=l2(regularizaion_weight)))\nadd_regularization_layer(model,\"batch_normalization\")\nadd_regularization_layer(model,\"dropout\", 0.5)\nmodel.add(Dense(num_clases, activation='sigmoid', kernel_regularizer=l2(regularizaion_weight)))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n# Documentation: https://keras.io/callbacks/","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Stops the training in the case of validation score not improving\nif early_stop_overfitting:\n    early_stop = EarlyStopping(patience=5)\n    \n    if not learning_rate_smoothing:\n        callbacks = [early_stop]\n\n# Reduces the learning rate of the back propagation gradient descend in the case of validation score not improving\nif learning_rate_smoothing:\n    learning_rate_reduction = ReduceLROnPlateau(monitor=\"val_acc\", patience=2, factor=learning_rate_reduction_factor, min_lr=0.00001, verbose=1)\n    \n    if not early_stop_overfitting:\n        callbacks = [learning_rate_reduction]\n    else:\n        callbacks = [early_stop, learning_rate_reduction]\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.summary()\n\nmodel.compile(loss='binary_crossentropy', optimizer=RMSprop(), metrics=['accuracy'])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if data_generation:\n    \n    history = model.fit_generator(data_generator, epochs=epochs, validation_data=val_generator, \n                                  steps_per_epoch=train_size//batch_size, validation_steps=val_size//batch_size, \n                                  callbacks=callbacks, verbose=2)\n\nelse:\n    \n    # Fourth dimension addition in case of its value being onesized\n    if img_channels == 1:\n        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2], 1)\n        X_val = X_val.reshape(X_val.shape[0], X_val.shape[1], X_val.shape[2], 1)\n\n    history = model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                        steps_per_epoch=train_size//batch_size, validation_steps=val_size//batch_size, \n                        callbacks=callbacks, verbose=2)\n\n    # Dimension restitution\n    if img_channels == 1:\n        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2])\n        X_val = X_val.reshape(X_val.shape[0], X_val.shape[1], X_val.shape[2])\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"evaluation\"></a>\n\n# Model result evaluation \nMatplotlib is used again, this time to plot the progression epoch after epoch of our model over metrics such as accuracy and loss, handling insight over the training progress, allowing its evaluation.\n\nHere, one of the most insightful aspects that shall be observed is if the model have suffered of overfitting, which means that our model is adapting way too much to the training set, which will make it way worse at generalizating (classifying data not present in the training dataset), which is precisely our objective.\n\nThe most obvious symptom of this is a growing difference in between the training and validation set's metric scores, most usually by the training set still improving over epochs while the validation set is stuck or even gets worse.\n\nMethods such as Data Augmentation and Dropout layers have been used for the model.","metadata":{}},{"cell_type":"code","source":"# summarize history for accuracy\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fourth dimension addition in case of its value being onesized\nif img_channels == 1:\n    X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2], 1)\n    \npredictions = model.predict(X_test, verbose=0)\n\n# Dimension restitution\nif img_channels == 1:\n    X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2])\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Preprocess test dataset showcase with predictions\nfor i in range(0, 60, 6): \n    plot_data(X_test[i:], predictions[i:], 6)\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"submission\"></a>\n# Results submission","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predictions array must be reshaped into a single dimension array in order to create the dataframe\npredictions = predictions.reshape(predictions.shape[0])\n\nif assign_test_labels:\n    labels = [1 if pred >= 0.5 else 0 for pred in predictions]\nelse:\n    labels = predictions\n\nsubmission = pd.DataFrame({'id':test_ids , 'label':labels})\n\n# Let's check what we've got\nsubmission.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"references\"></a>\n\n# References ","metadata":{}},{"cell_type":"markdown","source":"### Data augmentation\n\nhttps://machinelearningmastery.com/image-augmentation-deep-learning-keras/\n\nhttps://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html\n\n### Overfitting prevention\n\n#### Overall theorical approach\n\nhttps://machinelearningmastery.com/dropout-for-regularizing-deep-neural-networks/\n\n#### Dropout in Keras\n\nhttps://machinelearningmastery.com/how-to-reduce-overfitting-with-dropout-regularization-in-keras/\n\n#### Spatial Dropout\n\nhttps://towardsdatascience.com/review-tompson-cvpr15-spatial-dropout-human-pose-estimation-c7d6a5cecd8c\n\n#### Batch Normalization\n\nhttps://towardsdatascience.com/intuit-and-implement-batch-normalization-c05480333c5b\n\nhttps://towardsdatascience.com/dont-use-dropout-in-convolutional-networks-81486c823c16\n\nhttps://github.com/harrisonjansma/Research-Computer-Vision/blob/master/08-12-18%20Batch%20Norm%20vs%20Dropout/08-12-18%20Batch%20Norm%20vs%20Dropout.ipynb\n\n### References to other submissions\n\nhttps://www.kaggle.com/uysimty/keras-cnn-dog-or-cat-classification \n\n(From this other submission I got, both, the idea of using callbacks to improve model training, as well as more adecuate data augmentation parameters. Eventually, I as well got the idea of using batch normalization over the first dense layer in the classification part of the model.)\n\n### Very good end-to-end tutorials\n\nhttps://towardsdatascience.com/image-detection-from-scratch-in-keras-f314872006c9\n\nhttps://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html","metadata":{}}]}