{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"7baa365a-7329-3540-ebad-70769d4c23e5"},"source":"Was inspired by the Udacity Deep Learning Course.  Fairly new to Tensorflow so wanted to repurpose the NotMNIST-ConvNet from the course for this Cats & Dogs competition.  Unfortunately my accuracy is no better than the flip-of-a-coin so am hoping people will leave feedback on what I'm doing wrong.  Is a deeper (e.g. VGG-16) really required or did I miss something more fundamental?"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b50e1f7b-e2d1-d51a-0926-0f0cb37014d2"},"outputs":[],"source":"# These are all the modules we'll be using later. Make sure you can import them\n# before proceeding further.\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nfrom IPython.display import display, Image, HTML\nimport cv2\n\nTRAIN_DIR = '../input/train/'\nTEST_DIR = '../input/test/'"},{"cell_type":"markdown","metadata":{"_cell_guid":"90a7f0ca-de09-e332-1007-39f615b7de33"},"source":"- To run within a Kaggle Kernel, only use 2000 samples from TRAIN_DIR and 500 samples from TEST_DIR\n- Set image size to 96x96 since Kaggle Kernel was running out of memory with 224"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"11789326-57ed-0952-1aac-b7751fe8f2b5"},"outputs":[],"source":"# used for scaling/normalization\nIMAGE_SIZE = 150; # 150x150.  Also, 224, 96, 64, and 32 are also common\nCHANNELS = 3\npixel_depth = 255.0  # Number of levels per pixel.\n\n# for small-sample testing\nOUTFILE = '/Users/pal004/Desktop/CatsVsDogsRedux/CatsAndDogs_pal15Jan2017_SmallerTest.npsave.bin'\nTRAINING_AND_VALIDATION_SIZE_DOGS = 1000 \nTRAINING_AND_VALIDATION_SIZE_CATS = 1000 \nTRAINING_AND_VALIDATION_SIZE_ALL  = 2000\nTRAINING_SIZE = 1600  # TRAINING_SIZE + VALID_SIZE must equal TRAINING_AND_VALIDATION_SIZE_ALL\nVALID_SIZE = 400\nTEST_SIZE_ALL = 500\n\nif (TRAINING_SIZE + VALID_SIZE != TRAINING_AND_VALIDATION_SIZE_ALL):\n   print (\"Error, check that TRAINING_SIZE+VALID_SIZE is equal to TRAINING_AND_VALIDATION_SIZE_ALL\")\n   exit ()\n\ntrain_images = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)] \ntrain_dogs =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'dog' in i]\ntrain_cats =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'cat' in i]\ntest_images =  [TEST_DIR+i for i in os.listdir(TEST_DIR)]\n\ntrain_images = train_dogs[:TRAINING_AND_VALIDATION_SIZE_DOGS] + train_cats[:TRAINING_AND_VALIDATION_SIZE_CATS]\ntrain_labels = np.array ((['dogs'] * TRAINING_AND_VALIDATION_SIZE_DOGS) + (['cats'] * TRAINING_AND_VALIDATION_SIZE_CATS))\ntest_images =  test_images[:TEST_SIZE_ALL]\ntest_labels = np.array (['unknownclass'] * TEST_SIZE_ALL)"},{"cell_type":"markdown","metadata":{"_cell_guid":"e7a60070-b49e-1577-6209-6567a80c4239"},"source":"a.  Resize images to the same IMAGE_SIZE (96 x 96) set above\nb.  Don't change the aspect ratio of the image.  So if it doesn't fit in the 96x96 square, add 0-value padding to the right and bottom as appropriate\nc.  Normalize each of the color (R, B, G) layers indendently"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7aa44095-297e-84ea-82ec-1980b4f1fe76"},"outputs":[],"source":"# resizes to IMAGE_SIZE/IMAGE_SIZE while keeping aspect ratio the same.  pads on right/bottom as appropriate \ndef read_image(file_path):\n    img = cv2.imread(file_path, cv2.IMREAD_COLOR) #cv2.IMREAD_GRAYSCALE\n    if (img.shape[0] >= img.shape[1]): # height is greater than width\n       resizeto = (IMAGE_SIZE, int (round (IMAGE_SIZE * (float (img.shape[1])  / img.shape[0]))));\n    else:\n       resizeto = (int (round (IMAGE_SIZE * (float (img.shape[0])  / img.shape[1]))), IMAGE_SIZE);\n    \n    img2 = cv2.resize(img, (resizeto[1], resizeto[0]), interpolation=cv2.INTER_CUBIC)\n    img3 = cv2.copyMakeBorder(img2, 0, IMAGE_SIZE - img2.shape[0], 0, IMAGE_SIZE - img2.shape[1], cv2.BORDER_CONSTANT, 0)\n        \n    return img3[:,:,::-1]  # turn into rgb format\n\ndef prep_data(images):\n    count = len(images)\n    data = np.ndarray((count, IMAGE_SIZE, IMAGE_SIZE, CHANNELS), dtype=np.float32)\n\n    for i, image_file in enumerate(images):\n        image = read_image(image_file);\n        image_data = np.array (image, dtype=np.float32);\n        image_data[:,:,0] = (image_data[:,:,0].astype(float) - pixel_depth / 2) / pixel_depth\n        image_data[:,:,1] = (image_data[:,:,1].astype(float) - pixel_depth / 2) / pixel_depth\n        image_data[:,:,2] = (image_data[:,:,2].astype(float) - pixel_depth / 2) / pixel_depth\n        \n        data[i] = image_data; # image_data.T\n        if i%250 == 0: print('Processed {} of {}'.format(i, count))    \n    return data\n\ntrain_normalized = prep_data(train_images)\ntest_normalized = prep_data(test_images)\n\nprint(\"Train shape: {}\".format(train_normalized.shape))\nprint(\"Test shape: {}\".format(test_normalized.shape))"},{"cell_type":"markdown","metadata":{"_cell_guid":"afe1edec-a287-371b-6b0e-a93cc45f1346"},"source":"Just for visualization fun, print original image (first 3 dogs & first 3 cats) then image after resizing and normalization.  Is the normalization even correct?  "},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a4a83186-c361-4fb0-69a6-4aff689ea523"},"outputs":[],"source":"plt.imshow (train_normalized[0,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[1,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[2,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[1000,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[1001,:,:,:], interpolation='nearest')\nplt.figure ()\nplt.imshow (train_normalized[1002,:,:,:], interpolation='nearest')\n"},{"cell_type":"markdown","metadata":{"_cell_guid":"62fdfaa1-d9e3-37ee-78e4-6245dff3631a"},"source":"Randomize the samples from TRAIN_DIR and TEST_DIR.  Split the TRAIN_DIR samples for a train/validation split."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ddf26e09-36ea-d319-24fc-4f988bde26f1"},"outputs":[],"source":"np.random.seed (133)\ndef randomize(dataset, labels):\n  permutation = np.random.permutation(labels.shape[0])\n  shuffled_dataset = dataset[permutation,:,:,:]\n  shuffled_labels = labels[permutation]\n  return shuffled_dataset, shuffled_labels\n\ntrain_dataset_rand, train_labels_rand = randomize(train_normalized, train_labels)\ntest_dataset, test_labels = randomize(test_normalized, test_labels)\n\n# split up into training + valid\nvalid_dataset = train_dataset_rand[:VALID_SIZE,:,:,:]\nvalid_labels =   train_labels_rand[:VALID_SIZE]\ntrain_dataset = train_dataset_rand[VALID_SIZE:VALID_SIZE+TRAINING_SIZE,:,:,:]\ntrain_labels  = train_labels_rand[VALID_SIZE:VALID_SIZE+TRAINING_SIZE]\nprint ('Training', train_dataset.shape, train_labels.shape)\nprint ('Validation', valid_dataset.shape, valid_labels.shape)\nprint ('Test', test_dataset.shape, test_labels.shape)"},{"cell_type":"markdown","metadata":{"_cell_guid":"6f21ce0b-7f60-e1af-5379-de6b3b84aa70"},"source":"Start the TensorFlow portions and 1-hot-encode the labels"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7e8a7e12-d59f-efa0-2668-8216b592e24f"},"outputs":[],"source":"import tensorflow as tf\nimage_size = IMAGE_SIZE # TODO: redundant, consolidate\nnum_labels = 2\nnum_channels = 3 # rg\n\ndef reformat(dataset, labels):\n  dataset = dataset.reshape(\n    (-1, image_size, image_size, num_channels)).astype(np.float32)\n  labels = (labels=='cats').astype(np.float32); # set dogs to 0 and cats to 1\n  labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n  return dataset, labels\ntrain_dataset, train_labels = reformat(train_dataset, train_labels)\nvalid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\ntest_dataset, test_labels = reformat(test_dataset, test_labels)\nprint ('Training set', train_dataset.shape, train_labels.shape)\nprint ('Validation set', valid_dataset.shape, valid_labels.shape)\nprint ('Test set', test_dataset.shape, test_labels.shape)"},{"cell_type":"markdown","metadata":{"_cell_guid":"9e9e7059-0e31-f2d0-8176-32e04cebac09"},"source":"Define ConvNet Graph Model"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"6796b600-5add-2744-c043-87ada7bfa679"},"outputs":[],"source":"batch_size = 16\npatch_size = 5\ndepth = 16\nnum_hidden = 64\n\ngraph = tf.Graph()\n\nwith graph.as_default():\n\n  # Input data.\n  tf_train_dataset = tf.placeholder(\n    tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n  tf_valid_dataset = tf.constant(valid_dataset)\n  tf_test_dataset = tf.constant(test_dataset)\n\n  # variables \n  kernel_conv1 = tf.Variable(tf.truncated_normal([3, 3, 3, 32], dtype=tf.float32,\n                                            stddev=1e-1), name='weights_conv1')\n  biases_conv1 = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),\n                        trainable=True, name='biases_conv1')\n  kernel_conv2 = tf.Variable(tf.truncated_normal([3, 3, 32, 32], dtype=tf.float32,\n                                            stddev=1e-1), name='weights_conv2')\n  biases_conv2 = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),\n                        trainable=True, name='biases_conv2')\n  kernel_conv3 = tf.Variable(tf.truncated_normal([3, 3, 32, 64], dtype=tf.float32,\n                                            stddev=1e-1), name='weights_conv3')\n  biases_conv3 = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),\n                        trainable=True, name='biases_conv3')\n  fc1w = tf.Variable(tf.truncated_normal([23104, 64], \n                                                dtype=tf.float32,\n                                                stddev=1e-1), name='weights') # 23104 from pool3.gete_shape () of 19*19*64\n  fc1b = tf.Variable(tf.constant(1.0, shape=[64], dtype=tf.float32),\n                        trainable=True, name='biases')\n  fc2w = tf.Variable(tf.truncated_normal([64, 2],\n                                                dtype=tf.float32,\n                                                stddev=1e-1), name='weights')\n  fc2b = tf.Variable(tf.constant(1.0, shape=[2], dtype=tf.float32),\n                        trainable=True, name='biases')\n \n  \n  def model(data):\n     parameters = []\n     with tf.name_scope('conv1_1') as scope:\n         conv = tf.nn.conv2d(data, kernel_conv1, [1, 1, 1, 1], padding='SAME')\n         out = tf.nn.bias_add(conv, biases_conv1)\n         conv1_1 = tf.nn.relu(out, name=scope)\n         parameters += [kernel_conv1, biases_conv1]\n         \n     # pool1\n     pool1 = tf.nn.max_pool(conv1_1,\n                            ksize=[1, 2, 2, 1],\n                            strides=[1, 2, 2, 1],\n                            padding='SAME',\n                            name='pool1')\n     \n     with tf.name_scope('conv2_1') as scope:\n         conv = tf.nn.conv2d(pool1, kernel_conv2, [1, 1, 1, 1], padding='SAME')\n         out = tf.nn.bias_add(conv, biases_conv2)\n         conv2_1 = tf.nn.relu(out, name=scope)\n         parameters += [kernel_conv2, biases_conv2]\n         \n     # pool2\n     pool2 = tf.nn.max_pool(conv2_1,\n                            ksize=[1, 2, 2, 1],\n                            strides=[1, 2, 2, 1],\n                            padding='SAME',\n                            name='pool2')\n     \n     with tf.name_scope('conv3_1') as scope:\n         conv = tf.nn.conv2d(pool2, kernel_conv3, [1, 1, 1, 1], padding='SAME')\n         out = tf.nn.bias_add(conv, biases_conv3)\n         conv3_1 = tf.nn.relu(out, name=scope)\n         parameters += [kernel_conv3, biases_conv3]\n         \n     # pool3\n     pool3 = tf.nn.max_pool(conv3_1,\n                            ksize=[1, 2, 2, 1],\n                            strides=[1, 2, 2, 1],\n                            padding='SAME',\n                            name='pool3')\n         \n     # fc1\n     with tf.name_scope('fc1') as scope:\n         shape = int(np.prod(pool3.get_shape()[1:])) # except for batch size (the first one), multiple the dimensions\n         pool3_flat = tf.reshape(pool3, [-1, shape])\n         fc1l = tf.nn.bias_add(tf.matmul(pool3_flat, fc1w), fc1b)\n         fc1 = tf.nn.relu(fc1l)\n         parameters += [fc1w, fc1b]\n\n     # fc3\n     with tf.name_scope('fc3') as scope:\n         fc2l = tf.nn.bias_add(tf.matmul(fc1, fc2w), fc2b)\n         parameters += [fc2w, fc2b]\n     return fc2l;\n  \n  # Training computation.\n  logits = model(tf_train_dataset)\n  loss = tf.reduce_mean(\n    tf.nn.softmax_cross_entropy_with_logits(None, tf_train_labels, logits))\n    \n  # Optimizer.\n  optimizer = tf.train.RMSPropOptimizer(0.0001).minimize(loss)\n  \n  # Predictions for the training, validation, and test data.\n  train_prediction = tf.nn.softmax(logits)\n  valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n  test_prediction = tf.nn.softmax(model(tf_test_dataset))"},{"cell_type":"markdown","metadata":{"_cell_guid":"e462b95c-dac4-656f-5047-1a1bb4be4eff"},"source":"Take training data through graph and evaluate performance"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5b2e8f17-a494-54ed-bc5c-0d45aab9e58c"},"outputs":[],"source":"def accuracy(predictions, labels):\n   return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])\n\nnum_steps = 1001\nwith tf.Session(graph=graph) as session:\n  tf.initialize_all_variables().run()\n  print (\"Initialized\")\n  for step in range(num_steps):\n    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n    batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n    batch_labels = train_labels[offset:(offset + batch_size), :]\n    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n    _, l, predictions = session.run(\n      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n    if (step % 50 == 0):\n      print (\"Minibatch loss at step\", step, \":\", l)\n      print (\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n      print (\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n  #print (\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"6aaca0e8-7c7b-bb60-2dae-259fb4562ce0"},"outputs":[],"source":"tf_train_labels"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"bde56095-68a2-b19c-b406-413e63938d14"},"outputs":[],"source":"tf.nn.softmax_cross_entropy_with_logits(None, tf_train_labels, logits)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f0a691a2-b9f9-af80-0b6b-1968230d2bdc"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}