{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0,"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"7baa365a-7329-3540-ebad-70769d4c23e5","_active":false},"source":"Was inspired by the Udacity Deep Learning Course.  Fairly new to Tensorflow so wanted to repurpose the NotMNIST-ConvNet from the course for this Cats & Dogs competition.  This ConvNet gets >72% accuracy after only using a small fraction of the training data and very few epochs.  To prevent overfitting you should probably add (a) hinton dropout (b) perform data augmentation.  To provide better accuracy you can (a) train using all data  (b) increase # of epochs/training time  (c) build out full VGG-16 like architecture.","outputs":[]},{"cell_type":"code","execution_count":1,"metadata":{"_cell_guid":"b50e1f7b-e2d1-d51a-0926-0f0cb37014d2","_active":false},"outputs":[],"source":"# These are all the modules we'll be using later. Make sure you can import them\n# before proceeding further.\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nfrom IPython.display import display, Image, HTML\nimport cv2\n\nTRAIN_DIR = '../input/train/'\nTEST_DIR = '../input/test/'","execution_state":"idle"},{"cell_type":"markdown","metadata":{"_cell_guid":"90a7f0ca-de09-e332-1007-39f615b7de33","_active":false},"source":"- To run within a Kaggle Kernel, only use 2000 samples from TRAIN_DIR and 500 samples from TEST_DIR\n- Set image size to 96x96 since Kaggle Kernel was running out of memory with 224","outputs":[]},{"cell_type":"code","execution_count":2,"metadata":{"_cell_guid":"11789326-57ed-0952-1aac-b7751fe8f2b5","_active":false},"outputs":[],"source":"# used for scaling/normalization\nIMAGE_SIZE = 150; # 150x150.  Also, 224, 96, 64, and 32 are also common\nCHANNELS = 3\npixel_depth = 255.0  # Number of levels per pixel.\n\n# for small-sample testing\nOUTFILE = '/Users/pal004/Desktop/CatsVsDogsRedux/CatsAndDogs_pal15Jan2017_SmallerTest.npsave.bin'\nTRAINING_AND_VALIDATION_SIZE_DOGS = 1000 \nTRAINING_AND_VALIDATION_SIZE_CATS = 1000 \nTRAINING_AND_VALIDATION_SIZE_ALL  = 2000\nTRAINING_SIZE = 1600  # TRAINING_SIZE + VALID_SIZE must equal TRAINING_AND_VALIDATION_SIZE_ALL\nVALID_SIZE = 400\nTEST_SIZE_ALL = 500\n\nif (TRAINING_SIZE + VALID_SIZE != TRAINING_AND_VALIDATION_SIZE_ALL):\n   print (\"Error, check that TRAINING_SIZE+VALID_SIZE is equal to TRAINING_AND_VALIDATION_SIZE_ALL\")\n   exit ()\n\ntrain_images = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)] \ntrain_dogs =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'dog' in i]\ntrain_cats =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'cat' in i]\ntest_images =  [TEST_DIR+i for i in os.listdir(TEST_DIR)]\n\ntrain_images = train_dogs[:TRAINING_AND_VALIDATION_SIZE_DOGS] + train_cats[:TRAINING_AND_VALIDATION_SIZE_CATS]\ntrain_labels = np.array ((['dogs'] * TRAINING_AND_VALIDATION_SIZE_DOGS) + (['cats'] * TRAINING_AND_VALIDATION_SIZE_CATS))\ntest_images =  test_images[:TEST_SIZE_ALL]\ntest_labels = np.array (['unknownclass'] * TEST_SIZE_ALL)","execution_state":"idle"},{"cell_type":"markdown","metadata":{"_cell_guid":"e7a60070-b49e-1577-6209-6567a80c4239","_active":false},"source":"a.  Resize images to the same IMAGE_SIZE (150 x 150) set above\nb.  Don't change the aspect ratio of the image.  So if it doesn't fit in the 150x150 square, add 0-value padding to the right and bottom as appropriate\nc.  Normalize each of the color (R, B, G) layers indendently","outputs":[]},{"cell_type":"code","execution_count":3,"metadata":{"_cell_guid":"7aa44095-297e-84ea-82ec-1980b4f1fe76","_active":false},"outputs":[],"source":"# resizes to IMAGE_SIZE/IMAGE_SIZE while keeping aspect ratio the same.  pads on right/bottom as appropriate \ndef read_image(file_path):\n    img = cv2.imread(file_path, cv2.IMREAD_COLOR) #cv2.IMREAD_GRAYSCALE\n    if (img.shape[0] >= img.shape[1]): # height is greater than width\n       resizeto = (IMAGE_SIZE, int (round (IMAGE_SIZE * (float (img.shape[1])  / img.shape[0]))));\n    else:\n       resizeto = (int (round (IMAGE_SIZE * (float (img.shape[0])  / img.shape[1]))), IMAGE_SIZE);\n    \n    img2 = cv2.resize(img, (resizeto[1], resizeto[0]), interpolation=cv2.INTER_CUBIC)\n    img3 = cv2.copyMakeBorder(img2, 0, IMAGE_SIZE - img2.shape[0], 0, IMAGE_SIZE - img2.shape[1], cv2.BORDER_CONSTANT, 0)\n        \n    return img3[:,:,::-1]  # turn into rgb format\n\ndef prep_data(images):\n    count = len(images)\n    data = np.ndarray((count, IMAGE_SIZE, IMAGE_SIZE, CHANNELS), dtype=np.float32)\n\n    for i, image_file in enumerate(images):\n        image = read_image(image_file);\n        image_data = np.array (image, dtype=np.float32);\n        image_data[:,:,0] = (image_data[:,:,0].astype(float) - pixel_depth / 2) / pixel_depth\n        image_data[:,:,1] = (image_data[:,:,1].astype(float) - pixel_depth / 2) / pixel_depth\n        image_data[:,:,2] = (image_data[:,:,2].astype(float) - pixel_depth / 2) / pixel_depth\n        \n        data[i] = image_data; # image_data.T\n        if i%250 == 0: print('Processed {} of {}'.format(i, count))    \n    return data\n\ntrain_normalized = prep_data(train_images)\ntest_normalized = prep_data(test_images)\n\nprint(\"Train shape: {}\".format(train_normalized.shape))\nprint(\"Test shape: {}\".format(test_normalized.shape))","execution_state":"idle"},{"cell_type":"markdown","metadata":{"_cell_guid":"afe1edec-a287-371b-6b0e-a93cc45f1346","_active":false},"source":"Just for visualization fun, print original image (first 3 dogs & first 3 cats) then image after resizing and normalization.  ","outputs":[]},{"cell_type":"code","execution_count":4,"metadata":{"_cell_guid":"a4a83186-c361-4fb0-69a6-4aff689ea523","_active":false},"outputs":[],"source":"plt.imshow (train_normalized[0,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[1,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[2,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[1000,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[1001,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[1002,:,:,:], interpolation='nearest')","execution_state":"idle"},{"cell_type":"markdown","metadata":{"_cell_guid":"62fdfaa1-d9e3-37ee-78e4-6245dff3631a","_active":false},"source":"Randomize the samples from TRAIN_DIR and TEST_DIR.  Split the TRAIN_DIR samples for a train/validation split.","outputs":[]},{"cell_type":"code","execution_count":5,"metadata":{"_cell_guid":"ddf26e09-36ea-d319-24fc-4f988bde26f1","_active":false},"outputs":[],"source":"np.random.seed (133)\ndef randomize(dataset, labels):\n  permutation = np.random.permutation(labels.shape[0])\n  shuffled_dataset = dataset[permutation,:,:,:]\n  shuffled_labels = labels[permutation]\n  return shuffled_dataset, shuffled_labels\n\ntrain_dataset_rand, train_labels_rand = randomize(train_normalized, train_labels)\ntest_dataset, test_labels = randomize(test_normalized, test_labels)\n\n# split up into training + valid\nvalid_dataset = train_dataset_rand[:VALID_SIZE,:,:,:]\nvalid_labels =   train_labels_rand[:VALID_SIZE]\ntrain_dataset = train_dataset_rand[VALID_SIZE:VALID_SIZE+TRAINING_SIZE,:,:,:]\ntrain_labels  = train_labels_rand[VALID_SIZE:VALID_SIZE+TRAINING_SIZE]\nprint ('Training', train_dataset.shape, train_labels.shape)\nprint ('Validation', valid_dataset.shape, valid_labels.shape)\nprint ('Test', test_dataset.shape, test_labels.shape)","execution_state":"idle"},{"cell_type":"markdown","metadata":{"_cell_guid":"6f21ce0b-7f60-e1af-5379-de6b3b84aa70","_active":false},"source":"Start the TensorFlow portions and 1-hot-encode the labels","outputs":[]},{"cell_type":"code","execution_count":6,"metadata":{"_cell_guid":"7e8a7e12-d59f-efa0-2668-8216b592e24f","_active":false},"outputs":[],"source":"import tensorflow as tf\nimage_size = IMAGE_SIZE # TODO: redundant, consolidate\nnum_labels = 2\nnum_channels = 3 # rg\n\ndef reformat(dataset, labels):\n  dataset = dataset.reshape(\n    (-1, image_size, image_size, num_channels)).astype(np.float32)\n  labels = (labels=='cats').astype(np.float32); # set dogs to 0 and cats to 1\n  labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n  return dataset, labels\ntrain_dataset, train_labels = reformat(train_dataset, train_labels)\nvalid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\ntest_dataset, test_labels = reformat(test_dataset, test_labels)\nprint ('Training set', train_dataset.shape, train_labels.shape)\nprint ('Validation set', valid_dataset.shape, valid_labels.shape)\nprint ('Test set', test_dataset.shape, test_labels.shape)","execution_state":"idle"},{"cell_type":"markdown","metadata":{"_cell_guid":"9e9e7059-0e31-f2d0-8176-32e04cebac09","_active":false},"source":"Define ConvNet Graph Model","outputs":[]},{"cell_type":"code","execution_count":7,"metadata":{"_cell_guid":"6796b600-5add-2744-c043-87ada7bfa679","_active":true},"outputs":[],"source":"batch_size = 16\npatch_size = 5\ndepth = 16\nnum_hidden = 64\n\ngraph = tf.Graph()\n\nwith graph.as_default():\n\n  # Input data.\n  tf_train_dataset = tf.placeholder(\n    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n  tf_valid_dataset = tf.constant(valid_dataset)\n  tf_test_dataset = tf.constant(test_dataset)\n\n  # variables \n  kernel_conv1 = tf.Variable(tf.truncated_normal([3, 3, 3, 32], dtype=tf.float32,\n                                            stddev=1e-1), name='weights_conv1')\n  biases_conv1 = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),\n                        trainable=True, name='biases_conv1')\n  kernel_conv2 = tf.Variable(tf.truncated_normal([3, 3, 32, 32], dtype=tf.float32,\n                                            stddev=1e-1), name='weights_conv2')\n  biases_conv2 = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),\n                        trainable=True, name='biases_conv2')\n  kernel_conv3 = tf.Variable(tf.truncated_normal([3, 3, 32, 64], dtype=tf.float32,\n                                            stddev=1e-1), name='weights_conv3')\n  biases_conv3 = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),\n                        trainable=True, name='biases_conv3')\n  fc1w = tf.Variable(tf.truncated_normal([23104, 64], \n                                                dtype=tf.float32,\n                                                stddev=1e-1), name='weights') # 23104 from pool3.gete_shape () of 19*19*64\n  fc1b = tf.Variable(tf.constant(1.0, shape=[64], dtype=tf.float32),\n                        trainable=True, name='biases')\n  fc2w = tf.Variable(tf.truncated_normal([64, 2],\n                                                dtype=tf.float32,\n                                                stddev=1e-1), name='weights')\n  fc2b = tf.Variable(tf.constant(1.0, shape=[2], dtype=tf.float32),\n                        trainable=True, name='biases')\n \n  \n  def model(data):\n     parameters = []\n     with tf.name_scope('conv1_1') as scope:\n         conv = tf.nn.conv2d(data, kernel_conv1, [1, 1, 1, 1], padding='SAME')\n         out = tf.nn.bias_add(conv, biases_conv1)\n         conv1_1 = tf.nn.relu(out, name=scope)\n         parameters += [kernel_conv1, biases_conv1]\n         \n     # pool1\n     pool1 = tf.nn.max_pool(conv1_1,\n                            ksize=[1, 2, 2, 1],\n                            strides=[1, 2, 2, 1],\n                            padding='SAME',\n                            name='pool1')\n     \n     with tf.name_scope('conv2_1') as scope:\n         conv = tf.nn.conv2d(pool1, kernel_conv2, [1, 1, 1, 1], padding='SAME')\n         out = tf.nn.bias_add(conv, biases_conv2)\n         conv2_1 = tf.nn.relu(out, name=scope)\n         parameters += [kernel_conv2, biases_conv2]\n         \n     # pool2\n     pool2 = tf.nn.max_pool(conv2_1,\n                            ksize=[1, 2, 2, 1],\n                            strides=[1, 2, 2, 1],\n                            padding='SAME',\n                            name='pool2')\n     \n     with tf.name_scope('conv3_1') as scope:\n         conv = tf.nn.conv2d(pool2, kernel_conv3, [1, 1, 1, 1], padding='SAME')\n         out = tf.nn.bias_add(conv, biases_conv3)\n         conv3_1 = tf.nn.relu(out, name=scope)\n         parameters += [kernel_conv3, biases_conv3]\n         \n     # pool3\n     pool3 = tf.nn.max_pool(conv3_1,\n                            ksize=[1, 2, 2, 1],\n                            strides=[1, 2, 2, 1],\n                            padding='SAME',\n                            name='pool3')\n         \n     # fc1\n     with tf.name_scope('fc1') as scope:\n         shape = int(np.prod(pool3.get_shape()[1:])) # except for batch size (the first one), multiple the dimensions\n         pool3_flat = tf.reshape(pool3, [-1, shape])\n         fc1l = tf.nn.bias_add(tf.matmul(pool3_flat, fc1w), fc1b)\n         fc1 = tf.nn.relu(fc1l)\n         parameters += [fc1w, fc1b]\n\n     # fc3\n     with tf.name_scope('fc3') as scope:\n         fc2l = tf.nn.bias_add(tf.matmul(fc1, fc2w), fc2b)\n         parameters += [fc2w, fc2b]\n     return fc2l;\n  \n  # Training computation.\n  logits = model(tf_train_dataset)\n  loss = tf.reduce_mean(\n    tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n    \n  # Optimizer.\n  optimizer = tf.train.RMSPropOptimizer(0.0001).minimize(loss)\n  \n  # Predictions for the training, validation, and test data.\n  train_prediction = tf.nn.softmax(logits)\n  valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n  test_prediction = tf.nn.softmax(model(tf_test_dataset))","execution_state":"idle"},{"cell_type":"markdown","metadata":{"_cell_guid":"e462b95c-dac4-656f-5047-1a1bb4be4eff","_active":false},"source":"Take training data through graph and evaluate performance","outputs":[]},{"cell_type":"code","execution_count":8,"metadata":{"_cell_guid":"5b2e8f17-a494-54ed-bc5c-0d45aab9e58c","_active":false},"outputs":[],"source":"def accuracy(predictions, labels):\n   return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])\n\nnum_steps = 1001\nwith tf.Session(graph=graph) as session:\n  tf.initialize_all_variables().run()\n  print (\"Initialized\")\n  for step in range(num_steps):\n    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n    batch_labels = train_labels[offset:(offset + batch_size), :]\n    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n    _, l, predictions = session.run(\n      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n    if (step % 50 == 0):\n      print (\"Minibatch loss at step\", step, \":\", l)\n      print (\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n      print (\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n  #print (\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))","execution_state":"idle"},{"cell_type":"code","execution_count":9,"metadata":{"_cell_guid":"6aaca0e8-7c7b-bb60-2dae-259fb4562ce0","_active":false},"outputs":[],"source":null,"execution_state":"idle"}]}