{"nbformat_minor": 1, "nbformat": 4, "metadata": {"_is_fork": false, "language_info": {"name": "python", "version": "3.6.1", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "file_extension": ".py", "codemirror_mode": {"name": "ipython", "version": 3}, "mimetype": "text/x-python"}, "kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "_change_revision": 0}, "cells": [{"source": ["Was inspired by the Udacity Deep Learning Course.  Fairly new to Tensorflow so wanted to repurpose the NotMNIST-ConvNet from the course for this Cats & Dogs competition.  This ConvNet gets >72% accuracy after only using a small fraction of the training data and very few epochs.  To prevent overfitting you should probably add (a) hinton dropout (b) perform data augmentation.  To provide better accuracy you can (a) train using all data  (b) increase # of epochs/training time  (c) build out full VGG-16 like architecture."], "cell_type": "markdown", "metadata": {"_cell_guid": "7baa365a-7329-3540-ebad-70769d4c23e5", "_uuid": "9aaaa96310546ede333f7f4bb8758c19569abb17"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "b50e1f7b-e2d1-d51a-0926-0f0cb37014d2", "_uuid": "ddde91bd88db3e605321b9951025758a22907f0b"}, "source": ["# These are all the modules we'll be using later. Make sure you can import them\n", "# before proceeding further.\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import os\n", "from IPython.display import display, Image, HTML\n", "import cv2\n", "\n", "TRAIN_DIR = '../input/train/'\n", "TEST_DIR = '../input/test/'"]}, {"source": ["- To run within a Kaggle Kernel, only use 2000 samples from TRAIN_DIR and 500 samples from TEST_DIR\n", "- Set image size to 96x96 since Kaggle Kernel was running out of memory with 224"], "cell_type": "markdown", "metadata": {"_cell_guid": "90a7f0ca-de09-e332-1007-39f615b7de33", "_uuid": "8e9509ec5c821c9bfa65f859b924f8f31b5259a9"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"_cell_guid": "11789326-57ed-0952-1aac-b7751fe8f2b5", "_uuid": "859471e0182d1f2f518b70ee3b215cad2737d2e4"}, "source": ["# used for scaling/normalization\n", "IMAGE_SIZE = 150; # 150x150.  Also, 224, 96, 64, and 32 are also common\n", "CHANNELS = 3\n", "pixel_depth = 255.0  # Number of levels per pixel.\n", "\n", "# for small-sample testing\n", "OUTFILE = '/Users/pal004/Desktop/CatsVsDogsRedux/CatsAndDogs_pal15Jan2017_SmallerTest.npsave.bin'\n", "TRAINING_AND_VALIDATION_SIZE_DOGS = 1000 \n", "TRAINING_AND_VALIDATION_SIZE_CATS = 1000 \n", "TRAINING_AND_VALIDATION_SIZE_ALL  = 2000\n", "TRAINING_SIZE = 1600  # TRAINING_SIZE + VALID_SIZE must equal TRAINING_AND_VALIDATION_SIZE_ALL\n", "VALID_SIZE = 400\n", "TEST_SIZE_ALL = 500\n", "\n", "if (TRAINING_SIZE + VALID_SIZE != TRAINING_AND_VALIDATION_SIZE_ALL):\n", "   print (\"Error, check that TRAINING_SIZE+VALID_SIZE is equal to TRAINING_AND_VALIDATION_SIZE_ALL\")\n", "   exit ()\n", "\n", "train_images = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)] \n", "train_dogs =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'dog' in i]\n", "train_cats =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'cat' in i]\n", "test_images =  [TEST_DIR+i for i in os.listdir(TEST_DIR)]\n", "\n", "train_images = train_dogs[:TRAINING_AND_VALIDATION_SIZE_DOGS] + train_cats[:TRAINING_AND_VALIDATION_SIZE_CATS]\n", "train_labels = np.array ((['dogs'] * TRAINING_AND_VALIDATION_SIZE_DOGS) + (['cats'] * TRAINING_AND_VALIDATION_SIZE_CATS))\n", "test_images =  test_images[:TEST_SIZE_ALL]\n", "test_labels = np.array (['unknownclass'] * TEST_SIZE_ALL)"]}, {"source": ["a.  Resize images to the same IMAGE_SIZE (150 x 150) set above\n", "b.  Don't change the aspect ratio of the image.  So if it doesn't fit in the 150x150 square, add 0-value padding to the right and bottom as appropriate\n", "c.  Normalize each of the color (R, B, G) layers indendently"], "cell_type": "markdown", "metadata": {"_cell_guid": "e7a60070-b49e-1577-6209-6567a80c4239", "_uuid": "0aaa5685b4167797262970ab02fb4d5ff4a444b9"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "7aa44095-297e-84ea-82ec-1980b4f1fe76", "_uuid": "10fdcad4115a378692b8098f65541d4375f9caca"}, "source": ["# resizes to IMAGE_SIZE/IMAGE_SIZE while keeping aspect ratio the same.  pads on right/bottom as appropriate \n", "def read_image(file_path):\n", "    img = cv2.imread(file_path, cv2.IMREAD_COLOR) #cv2.IMREAD_GRAYSCALE\n", "    if (img.shape[0] >= img.shape[1]): # height is greater than width\n", "       resizeto = (IMAGE_SIZE, int (round (IMAGE_SIZE * (float (img.shape[1])  / img.shape[0]))));\n", "    else:\n", "       resizeto = (int (round (IMAGE_SIZE * (float (img.shape[0])  / img.shape[1]))), IMAGE_SIZE);\n", "    \n", "    img2 = cv2.resize(img, (resizeto[1], resizeto[0]), interpolation=cv2.INTER_CUBIC)\n", "    img3 = cv2.copyMakeBorder(img2, 0, IMAGE_SIZE - img2.shape[0], 0, IMAGE_SIZE - img2.shape[1], cv2.BORDER_CONSTANT, 0)\n", "        \n", "    return img3[:,:,::-1]  # turn into rgb format\n", "\n", "def prep_data(images):\n", "    count = len(images)\n", "    data = np.ndarray((count, IMAGE_SIZE, IMAGE_SIZE, CHANNELS), dtype=np.float32)\n", "\n", "    for i, image_file in enumerate(images):\n", "        image = read_image(image_file);\n", "        image_data = np.array (image, dtype=np.float32);\n", "        image_data[:,:,0] = (image_data[:,:,0].astype(float) - pixel_depth / 2) / pixel_depth\n", "        image_data[:,:,1] = (image_data[:,:,1].astype(float) - pixel_depth / 2) / pixel_depth\n", "        image_data[:,:,2] = (image_data[:,:,2].astype(float) - pixel_depth / 2) / pixel_depth\n", "        \n", "        data[i] = image_data; # image_data.T\n", "        if i%250 == 0: print('Processed {} of {}'.format(i, count))    \n", "    return data\n", "\n", "train_normalized = prep_data(train_images)\n", "test_normalized = prep_data(test_images)\n", "\n", "print(\"Train shape: {}\".format(train_normalized.shape))\n", "print(\"Test shape: {}\".format(test_normalized.shape))"]}, {"source": ["Just for visualization fun, print original image (first 3 dogs & first 3 cats) then image after resizing and normalization.  "], "cell_type": "markdown", "metadata": {"_cell_guid": "afe1edec-a287-371b-6b0e-a93cc45f1346", "_uuid": "cb48dbc26abb16c210defc4091bc0e65552e7519"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "a4a83186-c361-4fb0-69a6-4aff689ea523", "_uuid": "d5ec912c63703215a72a08b92c39332cf00c7317"}, "source": ["plt.imshow (train_normalized[0,:,:,:], interpolation='nearest')\n", "plt.figure ()\n", "plt.imshow (train_normalized[1,:,:,:], interpolation='nearest')\n", "plt.figure ()\n", "plt.imshow (train_normalized[2,:,:,:], interpolation='nearest')\n", "plt.figure ()\n", "plt.imshow (train_normalized[1000,:,:,:], interpolation='nearest')\n", "plt.figure ()\n", "plt.imshow (train_normalized[1001,:,:,:], interpolation='nearest')\n", "plt.figure ()\n", "plt.imshow (train_normalized[1002,:,:,:], interpolation='nearest')"]}, {"source": ["Randomize the samples from TRAIN_DIR and TEST_DIR.  Split the TRAIN_DIR samples for a train/validation split."], "cell_type": "markdown", "metadata": {"_cell_guid": "62fdfaa1-d9e3-37ee-78e4-6245dff3631a", "_uuid": "0a5994a4167f51a12f88347588b4805124c8df96"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "ddf26e09-36ea-d319-24fc-4f988bde26f1", "_uuid": "87930acf3e6daa39cb50d65c402fbcdbefa9c886"}, "source": ["np.random.seed (133)\n", "def randomize(dataset, labels):\n", "  permutation = np.random.permutation(labels.shape[0])\n", "  shuffled_dataset = dataset[permutation,:,:,:]\n", "  shuffled_labels = labels[permutation]\n", "  return shuffled_dataset, shuffled_labels\n", "\n", "train_dataset_rand, train_labels_rand = randomize(train_normalized, train_labels)\n", "test_dataset, test_labels = randomize(test_normalized, test_labels)\n", "\n", "# split up into training + valid\n", "valid_dataset = train_dataset_rand[:VALID_SIZE,:,:,:]\n", "valid_labels =   train_labels_rand[:VALID_SIZE]\n", "train_dataset = train_dataset_rand[VALID_SIZE:VALID_SIZE+TRAINING_SIZE,:,:,:]\n", "train_labels  = train_labels_rand[VALID_SIZE:VALID_SIZE+TRAINING_SIZE]\n", "print ('Training', train_dataset.shape, train_labels.shape)\n", "print ('Validation', valid_dataset.shape, valid_labels.shape)\n", "print ('Test', test_dataset.shape, test_labels.shape)"]}, {"source": ["Start the TensorFlow portions and 1-hot-encode the labels"], "cell_type": "markdown", "metadata": {"_cell_guid": "6f21ce0b-7f60-e1af-5379-de6b3b84aa70", "_uuid": "2408993c185a85edb768a37afd56f740bc9bab4f"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "7e8a7e12-d59f-efa0-2668-8216b592e24f", "_uuid": "5eaa4f30a192880dce3b7de21085805705131cc8"}, "source": ["import tensorflow as tf\n", "image_size = IMAGE_SIZE # TODO: redundant, consolidate\n", "num_labels = 2\n", "num_channels = 3 # rg\n", "\n", "def reformat(dataset, labels):\n", "  dataset = dataset.reshape(\n", "    (-1, image_size, image_size, num_channels)).astype(np.float32)\n", "  labels = (labels=='cats').astype(np.float32); # set dogs to 0 and cats to 1\n", "  labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n", "  return dataset, labels\n", "train_dataset, train_labels = reformat(train_dataset, train_labels)\n", "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n", "test_dataset, test_labels = reformat(test_dataset, test_labels)\n", "print ('Training set', train_dataset.shape, train_labels.shape)\n", "print ('Validation set', valid_dataset.shape, valid_labels.shape)\n", "print ('Test set', test_dataset.shape, test_labels.shape)"]}, {"source": ["Define ConvNet Graph Model"], "cell_type": "markdown", "metadata": {"_cell_guid": "9e9e7059-0e31-f2d0-8176-32e04cebac09", "_uuid": "5920d24568a92d99b9e6f467ae536d441d10a3da"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "6796b600-5add-2744-c043-87ada7bfa679", "_uuid": "d2027f9ceb03553ea7e6b8789d50b90c34bc9354"}, "source": ["batch_size = 16\n", "patch_size = 5\n", "depth = 16\n", "num_hidden = 64\n", "\n", "graph = tf.Graph()\n", "\n", "with graph.as_default():\n", "\n", "  # Input data.\n", "  tf_train_dataset = tf.placeholder(\n", "    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n", "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n", "  tf_valid_dataset = tf.constant(valid_dataset)\n", "  tf_test_dataset = tf.constant(test_dataset)\n", "\n", "  # variables \n", "  kernel_conv1 = tf.Variable(tf.truncated_normal([3, 3, 3, 32], dtype=tf.float32,\n", "                                            stddev=1e-1), name='weights_conv1')\n", "  biases_conv1 = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),\n", "                        trainable=True, name='biases_conv1')\n", "  kernel_conv2 = tf.Variable(tf.truncated_normal([3, 3, 32, 32], dtype=tf.float32,\n", "                                            stddev=1e-1), name='weights_conv2')\n", "  biases_conv2 = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),\n", "                        trainable=True, name='biases_conv2')\n", "  kernel_conv3 = tf.Variable(tf.truncated_normal([3, 3, 32, 64], dtype=tf.float32,\n", "                                            stddev=1e-1), name='weights_conv3')\n", "  biases_conv3 = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),\n", "                        trainable=True, name='biases_conv3')\n", "  fc1w = tf.Variable(tf.truncated_normal([23104, 64], \n", "                                                dtype=tf.float32,\n", "                                                stddev=1e-1), name='weights') # 23104 from pool3.gete_shape () of 19*19*64\n", "  fc1b = tf.Variable(tf.constant(1.0, shape=[64], dtype=tf.float32),\n", "                        trainable=True, name='biases')\n", "  fc2w = tf.Variable(tf.truncated_normal([64, 2],\n", "                                                dtype=tf.float32,\n", "                                                stddev=1e-1), name='weights')\n", "  fc2b = tf.Variable(tf.constant(1.0, shape=[2], dtype=tf.float32),\n", "                        trainable=True, name='biases')\n", " \n", "  \n", "  def model(data):\n", "     parameters = []\n", "     with tf.name_scope('conv1_1') as scope:\n", "         conv = tf.nn.conv2d(data, kernel_conv1, [1, 1, 1, 1], padding='SAME')\n", "         out = tf.nn.bias_add(conv, biases_conv1)\n", "         conv1_1 = tf.nn.relu(out, name=scope)\n", "         parameters += [kernel_conv1, biases_conv1]\n", "         \n", "     # pool1\n", "     pool1 = tf.nn.max_pool(conv1_1,\n", "                            ksize=[1, 2, 2, 1],\n", "                            strides=[1, 2, 2, 1],\n", "                            padding='SAME',\n", "                            name='pool1')\n", "     \n", "     with tf.name_scope('conv2_1') as scope:\n", "         conv = tf.nn.conv2d(pool1, kernel_conv2, [1, 1, 1, 1], padding='SAME')\n", "         out = tf.nn.bias_add(conv, biases_conv2)\n", "         conv2_1 = tf.nn.relu(out, name=scope)\n", "         parameters += [kernel_conv2, biases_conv2]\n", "         \n", "     # pool2\n", "     pool2 = tf.nn.max_pool(conv2_1,\n", "                            ksize=[1, 2, 2, 1],\n", "                            strides=[1, 2, 2, 1],\n", "                            padding='SAME',\n", "                            name='pool2')\n", "     \n", "     with tf.name_scope('conv3_1') as scope:\n", "         conv = tf.nn.conv2d(pool2, kernel_conv3, [1, 1, 1, 1], padding='SAME')\n", "         out = tf.nn.bias_add(conv, biases_conv3)\n", "         conv3_1 = tf.nn.relu(out, name=scope)\n", "         parameters += [kernel_conv3, biases_conv3]\n", "         \n", "     # pool3\n", "     pool3 = tf.nn.max_pool(conv3_1,\n", "                            ksize=[1, 2, 2, 1],\n", "                            strides=[1, 2, 2, 1],\n", "                            padding='SAME',\n", "                            name='pool3')\n", "         \n", "     # fc1\n", "     with tf.name_scope('fc1') as scope:\n", "         shape = int(np.prod(pool3.get_shape()[1:])) # except for batch size (the first one), multiple the dimensions\n", "         pool3_flat = tf.reshape(pool3, [-1, shape])\n", "         fc1l = tf.nn.bias_add(tf.matmul(pool3_flat, fc1w), fc1b)\n", "         fc1 = tf.nn.relu(fc1l)\n", "         parameters += [fc1w, fc1b]\n", "\n", "     # fc3\n", "     with tf.name_scope('fc3') as scope:\n", "         fc2l = tf.nn.bias_add(tf.matmul(fc1, fc2w), fc2b)\n", "         parameters += [fc2w, fc2b]\n", "     return fc2l;\n", "  \n", "  # Training computation.\n", "  logits = model(tf_train_dataset)\n", "  loss = tf.reduce_mean(\n", "    tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n", "    \n", "  # Optimizer.\n", "  optimizer = tf.train.RMSPropOptimizer(0.0001).minimize(loss)\n", "  \n", "  # Predictions for the training, validation, and test data.\n", "  train_prediction = tf.nn.softmax(logits)\n", "  valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n", "  test_prediction = tf.nn.softmax(model(tf_test_dataset))"]}, {"source": ["Take training data through graph and evaluate performance"], "cell_type": "markdown", "metadata": {"_cell_guid": "e462b95c-dac4-656f-5047-1a1bb4be4eff", "_uuid": "b2295a986e18dcb50926630b254cdc4a4bbcd2a6"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "5b2e8f17-a494-54ed-bc5c-0d45aab9e58c", "_uuid": "f9bf83f530db9d688e60046aaa39400ed17e04fa"}, "source": ["def accuracy(predictions, labels):\n", "   return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])\n", "\n", "num_steps = 1001\n", "with tf.Session(graph=graph) as session:\n", "  tf.initialize_all_variables().run()\n", "  print (\"Initialized\")\n", "  for step in range(num_steps):\n", "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n", "    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n", "    batch_labels = train_labels[offset:(offset + batch_size), :]\n", "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n", "    _, l, predictions = session.run(\n", "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n", "    if (step % 50 == 0):\n", "      print (\"Minibatch loss at step\", step, \":\", l)\n", "      print (\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n", "      print (\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n", "  #print (\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))"]}, {"outputs": [], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "6aaca0e8-7c7b-bb60-2dae-259fb4562ce0", "_uuid": "73b31718e1315b075b87d8e29655cd904deee656"}, "source": []}]}