{
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0,
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "77500d44-2eae-e515-07f8-a77344b3672f",
        "_active": false
      },
      "source": "Starter EDA and ConvNet implementation using Keras. \n\nInspiration for this notebook comes from this [Keras blog post](https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html) and the [VGG ConvNet paper](https://arxiv.org/pdf/1409.1556.pdf). ",
      "execution_count": null,
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "a9757904-1596-c427-c416-0c8c4b9b2e69",
        "_active": false
      },
      "outputs": [],
      "source": "import os, cv2, random\nimport numpy as np\nimport pandas as pd",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "fcc0afc3-683d-aa32-bf13-aadb33d1f2a5",
        "_active": false
      },
      "outputs": [],
      "source": "import theano",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "b610b329-d5be-11f7-f40c-fd502eea0854",
        "_active": false
      },
      "outputs": [],
      "source": "from keras import backend as K\nK.image_dim_ordering()",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "25823f66-6cab-e06d-1aff-91672ab04be6",
        "_active": false
      },
      "outputs": [],
      "source": "K.set_image_dim_ordering('th')",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "24f8b5fe-16e9-b7a2-43d4-120404df8fe1",
        "_active": false
      },
      "outputs": [],
      "source": "import matplotlib.pyplot as plt\nfrom matplotlib import ticker\nimport seaborn as sns\n%matplotlib inline \n\nfrom keras.models import Sequential\nfrom keras.layers import Input, Dropout, Flatten, Convolution2D, MaxPooling2D, Dense, Activation\nfrom keras.optimizers import RMSprop, SGD, Adam, adadelta\nfrom keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\nfrom keras.utils import np_utils",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "72f96515-d7c9-3742-b60f-025368c59a6a",
        "_active": false
      },
      "outputs": [],
      "source": "TRAIN_DIR = '../input/train/'\nTEST_DIR = '../input/test/'\n\nROWS = 64\nCOLS = 64\nCHANNELS = 3\n\n#train_images = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)] # use this for full dataset\ntrain_dogs =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'dog' in i]\ntrain_cats =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'cat' in i]\n\ntest_images =  [TEST_DIR+i for i in os.listdir(TEST_DIR)]\n\n\n# slice datasets for memory efficiency on Kaggle Kernels, delete if using full dataset\ntrain_images = train_dogs[:10] + train_cats[:10]\nval_images=train_dogs[10:20] + train_cats[10:20]\nrandom.shuffle(train_images)\ntest_images =  test_images[:10]\n\ndef read_image(file_path):\n    \n    img=cv2.resize(cv2.imread(file_path), (ROWS, COLS), interpolation=cv2.INTER_CUBIC)\n    img=img.transpose((2,0,1)) #use this if you want the image to be (channel, row, cols)\n    return img\n\ndef prep_data(images):\n    count = len(images)\n    data = np.ndarray((count,CHANNELS, ROWS, COLS ))\n\n    for i, image_file in enumerate(images):\n        image = read_image(image_file)\n        data[i] = image\n        if i%250 == 0: print('Processed {} of {}'.format(i, count))\n    \n    return data\n\ntrain = prep_data(train_images)\ntest = prep_data(test_images)\n\nprint(\"Train shape: {}\".format(train.shape))\nprint(\"Test shape: {}\".format(test.shape))",
      "execution_state": "idle"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d515ab85-9bfe-b65b-6b48-2873b2f03fd2",
        "_active": false
      },
      "source": "### Generating the Labels\n\nWe're dealing with a binary classification problem here - (1) dog (0) cat. The lables can be created by looping over the file names in the train directory. It's nice to see the training data is perfectly balanced. ",
      "execution_count": null,
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "57a3e50e-23eb-7b05-b134-4724cbe46317",
        "_active": false
      },
      "outputs": [],
      "source": "labels = []\nfor i in train_images:\n    if 'dog' in i:\n        labels.append(1)\n    else:\n        labels.append(0)\n        \nlabels_test = []\nfor i in test_images:\n    if 'dog' in i:\n        labels_test.append(1)\n    else:\n        labels_test.append(0)\n        ",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "034ee1b9-7dee-c072-7d4f-ea2c06f7683f",
        "_active": false
      },
      "outputs": [],
      "source": "optimizer = RMSprop(1e-4)\n#logloss-> binary_crossentropy\nobjective = 'binary_crossentropy'\n\n#set dim_ordering to 'th' so it reads channel dimension at index 1\ndef catdog():\n    \n    model = Sequential()\n\n    model.add(Convolution2D(32, 3, 3, border_mode='same', input_shape=(3, ROWS, COLS),activation='relu'))\n    model.add(Convolution2D(32, 3, 3, border_mode='same' ,activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))\n    model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(Convolution2D(128, 3, 3, border_mode='same', activation='relu'))\n    model.add(Convolution2D(128, 3, 3, border_mode='same', activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(Convolution2D(256, 3, 3, border_mode='same', activation='relu'))\n    model.add(Convolution2D(256, 3, 3, border_mode='same',activation='relu'))\n#     model.add(Convolution2D(256, 3, 3, border_mode='same', activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n#     model.add(Convolution2D(256, 3, 3, border_mode='same', activation='relu'))\n#     model.add(Convolution2D(256, 3, 3, border_mode='same', activation='relu'))\n#     model.add(Convolution2D(256, 3, 3, border_mode='same', activation='relu'))\n#     model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Flatten())\n    model.add(Dense(256, activation='relu'))\n    model.add(Dropout(0.5))\n    \n    model.add(Dense(256, activation='relu'))\n    model.add(Dropout(0.5))\n\n    model.add(Dense(1))\n    model.add(Activation('sigmoid'))\n    \n    model.compile(loss=objective, optimizer=optimizer, metrics=['accuracy'])\n    \n    return model\n\n\nmodel = catdog()",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "e4e30c4d-d4b5-a161-66bb-ea0395916750",
        "_active": false
      },
      "outputs": [],
      "source": "nb_epoch = 3\nbatch_size = 5\n\nfrom keras.callbacks import History \nhistory = History()\nearly_stopping=EarlyStopping(monitor='val_loss', patience=1)\ntrain_model=model.fit(train, labels, batch_size=batch_size, nb_epoch=nb_epoch,validation_split=0.1, verbose=1, shuffle=True, callbacks=[history, early_stopping])\n    ",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "6c5b7f3d-e985-90bc-8a9d-851a8bc17375",
        "_active": false
      },
      "outputs": [],
      "source": "# train the model on the new data for a few epochs",
      "execution_state": "idle"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "5c7c3d50-1c2b-9336-f461-b500cc2fb45c",
        "_active": false
      },
      "source": "### Checking out Cats and Dogs\nA quick side-by-side comparison of the animals.",
      "execution_count": null,
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "c2354464-140c-49b4-52ee-d35b017240c4",
        "_active": false
      },
      "source": "### Your Average Cat and Dog Photo\n\nJust for fun, the mean pixel values for cats and dogs. I can almost see a resemblance, a ghost, if you will...",
      "execution_count": null,
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "f48cfd0a-a704-271e-2321-81c4f7b32c95",
        "_active": false
      },
      "source": "## CatdogNet-16\n\nA scaled down version of the VGG-16, with a few notable changes.\n\n- Number of convolution filters cut in half, fully connected (dense) layers scaled down. \n- Optimizer changed to `RMSprop`. \n- Output layer activation set to `sigmoid` for binary crossentropy. \n- Some layers commented out for efficiency.\n\nThe full network takes about 80s per epoch on a GTX1070 (or 2hr+ on CPU) on the full dataset.  (This script only trains on 8% of the 25K images. )",
      "execution_count": null,
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "eae46fb3-3633-f12f-3ef6-4984cdb5d85a",
        "_active": false
      },
      "source": "### Train and Predict\n\nI'm using Keras's early stopping callback to end training when the validation loss stops improving, otherwise the model will overfit. I will also be tracking the loss history on each epoch to visualize the overfitting trend. \n\nNote: A slice of 1000 images was used to fit the model for CPU efficency. The model's perfrmance improves significantly when used on the entire dataset. ",
      "execution_count": null,
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "8c2c0147-e7e7-be89-5e41-7afdd0ee221a",
        "_active": false
      },
      "outputs": [],
      "source": "predictions = model.predict(test, verbose=0)\nprint(predictions)",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "550b9e66-7707-589f-0e45-d43eded1c3b4",
        "_active": false
      },
      "outputs": [],
      "source": "np.mean(train_model.history['val_loss'])",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "a7d31db7-7a5f-0f82-852a-a278009ede44",
        "_active": false
      },
      "outputs": [],
      "source": "plt.plot(train_model.history['loss'], 'blue', label='Training Loss')\nplt.plot(train_model.history['val_loss'], 'green', label='Validation Loss')\nplt.xlabel('epoch')\nplt.ylabel('log loss')\nplt.legend(loc='upper right')\nplt.show()",
      "execution_state": "idle"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "3f1134a8-8fda-e4a8-7429-676b79276f5f",
        "_active": false
      },
      "source": "## How'd We Do?\n\nI'm pretty sure I can distinguish a cat from a dog 100% of the time, but how confident is the model?...\n\nTip: Run on the full dataset with a GPU for a LB logloss of ~0.4 and accuracy at approx 90%. ",
      "execution_count": null,
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "7bce51e1-c218-209d-73e4-b9002924d5e3",
        "_active": false
      },
      "outputs": [],
      "source": "for i in range(0,10):\n    if predictions[i, 0] >= 0.5: \n        print('I am {:.2%} sure this is a Dog'.format(predictions[i][0]))\n    else: \n        print('I am {:.2%} sure this is a Cat'.format(1-predictions[i][0]))\n        \n    plt.imshow(test[i].T)\n    plt.show()",
      "execution_state": "idle"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "a4a77d59-8234-edc3-c0aa-25e8a424bdcd",
        "_active": false
      },
      "source": "# try image preprocessing",
      "execution_count": null,
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "f61a2405-1ff3-8bce-05ed-3724c9c426c0",
        "_active": false
      },
      "outputs": [],
      "source": "from keras.preprocessing.image import ImageDataGenerator",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "bd8a849e-1bcb-3ccc-376e-98b829fa70c9",
        "_active": false
      },
      "outputs": [],
      "source": "datagen = ImageDataGenerator(\n    featurewise_center=True,\n    featurewise_std_normalization=True,\n    horizontal_flip=True)",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "ed32d76d-8384-7740-add0-b3d46fb59c53",
        "_active": false
      },
      "outputs": [],
      "source": "\n# compute quantities required for featurewise normalization\n# (std, mean, and principal components if ZCA whitening is applied)\ndatagen.fit(train)",
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "3a74e368-1d78-e0f2-56c6-2ceb9a2c440f",
        "_active": false,
        "collapsed": false
      },
      "source": "X_train = train.reshape(train.shape[0], 3, 64, 64)",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "9e1bce4f-7914-5a16-c681-da3210f3e573",
        "_active": false,
        "collapsed": false
      },
      "source": "x = img_to_array(img)\nx = x.reshape((1,) + x.shape)",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "7a54ae89-f7b3-d11a-5208-e9bc15972c14",
        "_active": false,
        "collapsed": false
      },
      "source": "X= datagen.flow(train, labels, batch_size=5)",
      "execution_count": 31,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "44e05a01-8f70-1f51-df8b-3fc60fd39e43",
        "_active": false,
        "collapsed": false
      },
      "source": null,
      "execution_count": 21,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "collapsed": false,
        "_cell_guid": "3c7a2210-32ef-3178-2e6d-8e3d8dff20b0",
        "_active": true
      },
      "outputs": [],
      "source": "for e in range(nb_epoch):\n    print 'Epoch', e\n    batches = 0\n    for X_batch, Y_batch in datagen.flow(train, labels, batch_size=10):\n        loss = model.train(X_batch, Y_batch)\n        batches += 1\n        if batches >= len(train) / 10:\n            # we need to break the loop by hand because\n            # the generator loops indefinitely\n            break",
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "9679de49-2d2d-1a8a-bd0d-523cd5fab45e",
        "_active": false,
        "collapsed": false
      },
      "source": null,
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    }
  ]
}