{"cells": [{"metadata": {"_uuid": "0d198e225f287ebc77f2da61fd0b1fca70734aba", "collapsed": true, "_cell_guid": "d1f5c103-29ae-4842-ab6e-9101211a8e2b"}, "outputs": [], "cell_type": "code", "source": ["import numpy as np\n", "import pandas as pd\n", "import tensorflow as tf\n", "import os, random, cv2\n", "import math\n", "\n", "%matplotlib inline\n", "import matplotlib.pyplot as plt\n", "import matplotlib.cm as cm\n", "\n", "TRAIN_DIR = '../input/train/'\n", "TEST_DIR = '../input/test/'\n", "\n", "# image number to output\n", "IMAGE_TO_DISPLAY = 4\n", "\n", "IMAGE_SIZE = 64\n", "IMG_ROW = 64\n", "IMG_COL = 64\n", "IMG_CHA = 3\n", "\n", "LEARNING_RATE = 1e-10\n", "TRAINING_ITERATIONS = 500\n", "#EXTRA_ITERATIONS = 4000\n", "\n", "DROPOUT = .5\n", "BATCH_SIZE = 50\n", "#EXTRA_BATCH_SIZE = 400\n", "\n", "TRAIN_SIZE = 100 #Set to 25000 for entire set, validation is cut from here\n", "TEST_SIZE = 50 #Set to 12500 for entire set\n", "VALIDATION_SIZE = 50\n", "\n", "FILTER_SIZE = 5 #Produces NxN filters\n", "\n", "FILTER_NUM_1 = 32 #Number of filters at given layer\n", "FILTER_NUM_2 = 32\n", "FILTER_NUM_3 = 64\n", "FILTER_NUM_4 = 64\n", "FILTER_NUM_5 = 128\n", "FILTER_NUM_6 = 128\n", "\n", "train_images_dir = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)]\n", "train_dogs_dir =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'dog' in i]\n", "train_cats_dir =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'cat' in i]\n", "test_images_dir =  [TEST_DIR+i for i in os.listdir(TEST_DIR)]\n", "\n", "random.shuffle(train_images_dir)"], "execution_count": 1}, {"metadata": {"_uuid": "9e181629a809958e30e113415d53ef3c0588ce59", "collapsed": true, "_cell_guid": "e454d3d9-89d8-47ac-a227-6f34fabdf932"}, "outputs": [], "cell_type": "code", "source": ["def read_image(file_path):\n", "    img = cv2.imread(file_path, cv2.IMREAD_COLOR)\n", "    if (img.shape[0] >= img.shape[1]): # height is greater than width\n", "       resizeto = (IMAGE_SIZE, int (round (IMAGE_SIZE * (float (img.shape[1])  / img.shape[0]))));\n", "    else:\n", "       resizeto = (int (round (IMAGE_SIZE * (float (img.shape[0])  / img.shape[1]))), IMAGE_SIZE);\n", "    \n", "    img2 = cv2.resize(img, (resizeto[1], resizeto[0]), interpolation=cv2.INTER_CUBIC)\n", "    img3 = cv2.copyMakeBorder(img2, 0, IMAGE_SIZE - img2.shape[0], 0, IMAGE_SIZE - img2.shape[1], cv2.BORDER_CONSTANT, 0)\n", "        \n", "    return img3[:,:,::-1]  # turn into rgb format\n", "\n", "def prep_data(images):\n", "    count = len(images)\n", "    data = np.ndarray((count, IMAGE_SIZE, IMAGE_SIZE, IMG_CHA), dtype=np.float32)\n", "    labels = np.zeros((len(images),2))\n", "\n", "    for i, image_file in enumerate(images):\n", "        image = read_image(image_file);\n", "        image_data = np.array (image, dtype=np.float32);\n", "        image_data[:,:,0] = (image_data[:,:,0].astype(float)) / 255-.5\n", "        image_data[:,:,1] = (image_data[:,:,1].astype(float)) / 255-.5\n", "        image_data[:,:,2] = (image_data[:,:,2].astype(float)) / 255-.5\n", "        if('/dog' in image_file):\n", "            labels[i,0] = 1\n", "        else:\n", "            labels[i,1] = 1\n", "        \n", "        data[i] = image_data; # image_data.T\n", "        if i%250 == 0 or (i+1)==count: print('Processed {} of {}'.format(i, count))    \n", "    data = np.resize(data.T, [len(images),(IMG_ROW*IMG_COL), 3])\n", "    return data, labels"], "execution_count": 2}, {"metadata": {"_uuid": "08de0a947f0cd86ac18f68ed87cb604c3d740175", "_cell_guid": "4a86e8e7-44ac-4f71-94ab-f941c0416240"}, "outputs": [], "cell_type": "code", "source": ["train, labels = prep_data(train_images_dir[:TRAIN_SIZE])\n", "test, test_labels = prep_data(test_images_dir[:TEST_SIZE])\n", "# split data into training & validation\n", "validation_images = train[:VALIDATION_SIZE]\n", "validation_labels = labels[:VALIDATION_SIZE]\n", "\n", "train_images = train[VALIDATION_SIZE:]\n", "train_labels = labels[VALIDATION_SIZE:]"], "execution_count": 3}, {"metadata": {"_uuid": "2565e8fad7a26f3dcd4f25f4d1a7c897fca407c3", "collapsed": true, "_cell_guid": "d81d9035-8250-4f6f-8158-7c5b185d00d0"}, "outputs": [], "cell_type": "code", "source": ["# xavier initialization\n", "def weight_variable(shape,n_in,n_out):\n", "    initial = tf.truncated_normal(shape, stddev=math.sqrt(2. / (n_in + n_out)))\n", "    return tf.Variable(initial)\n", "\n", "def bias_variable(shape,n_in,n_out):\n", "    initial = tf.constant(math.sqrt(2. / (n_in + n_out)), shape=shape)\n", "    return tf.Variable(initial)\n", "\n", "# convolution\n", "def conv2d(x, W):\n", "    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n", "\n", "def max_pool_2x2(x):\n", "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n", "\n", "# input & output of NN\n", "\n", "# images\n", "x = tf.placeholder('float', shape=[None, train.shape[1], train.shape[2]], )\n", "# labels\n", "y_ = tf.placeholder('float', shape=[None, 2])\n", "\n", "def lrelu(x):\n", "  return tf.nn.relu(x) - .01 * tf.nn.relu(-x)\n", "\n", "image_size = train.shape[1]\n", "image_width = image_height = np.ceil(np.sqrt(image_size)).astype(np.uint8)"], "execution_count": 4}, {"metadata": {"_uuid": "e4bea2f3ec9bbc6b758f37c85e0ba599caab027d", "collapsed": true, "_cell_guid": "51f5ba19-0cc6-4d4e-a0e9-6aacbaea9e41"}, "outputs": [], "cell_type": "code", "source": ["# first convolutional layer\n", "W_conv1 = weight_variable([FILTER_SIZE, FILTER_SIZE, IMG_CHA, FILTER_NUM_1], IMG_CHA, FILTER_NUM_1)\n", "b_conv1 = bias_variable([FILTER_NUM_1], IMG_CHA, FILTER_NUM_1)\n", "\n", "image = tf.reshape(x, [-1, image_width, image_height, IMG_CHA])\n", "\n", "h_conv1 = lrelu(conv2d(image, W_conv1) + b_conv1)\n", "h_pool1 = h_conv1"], "execution_count": 5}, {"metadata": {"_uuid": "cbc552d59da41e85ead915e5dec8e124200c313a", "collapsed": true, "_cell_guid": "c8408e9a-d6af-4a8f-8f01-1efda818a21e"}, "outputs": [], "cell_type": "code", "source": ["# second convolutional layer\n", "W_conv2 = weight_variable([FILTER_SIZE, FILTER_SIZE, FILTER_NUM_1, FILTER_NUM_2], FILTER_NUM_1, FILTER_NUM_2)\n", "b_conv2 = bias_variable([FILTER_NUM_2], FILTER_NUM_1, FILTER_NUM_2)\n", "\n", "h_conv2 = lrelu(conv2d(h_pool1, W_conv2) + b_conv2)\n", "h_pool2 = max_pool_2x2(h_conv2)"], "execution_count": 6}, {"metadata": {"_uuid": "1effec9547d75bea70462e0ee8f741e30535f75e", "collapsed": true, "_cell_guid": "f73dea3f-7e74-4f4c-89f3-2003f2f40cb6"}, "outputs": [], "cell_type": "code", "source": ["# third convolutional layer\n", "W_conv3 = weight_variable([FILTER_SIZE, FILTER_SIZE, FILTER_NUM_2, FILTER_NUM_3], FILTER_NUM_2, FILTER_NUM_3)\n", "b_conv3 = bias_variable([FILTER_NUM_3], FILTER_NUM_2, FILTER_NUM_3)\n", "\n", "h_conv3 = lrelu(conv2d(h_pool2, W_conv3) + b_conv3)\n", "h_pool3 = h_conv3"], "execution_count": 7}, {"metadata": {"_uuid": "40f1db1029b184b38e4bfa1f12a3b60435a13ff8", "collapsed": true, "_cell_guid": "84c30555-92f3-4675-864e-48c5d3e1ea45"}, "outputs": [], "cell_type": "code", "source": ["# fourth convolutional layer\n", "W_conv4 = weight_variable([FILTER_SIZE, FILTER_SIZE, FILTER_NUM_3, FILTER_NUM_4], FILTER_NUM_3, FILTER_NUM_4)\n", "b_conv4 = bias_variable([FILTER_NUM_4], FILTER_NUM_3, FILTER_NUM_4)\n", "\n", "h_conv4 = lrelu(conv2d(h_pool3, W_conv4) + b_conv4)\n", "h_pool4 = max_pool_2x2(h_conv4)"], "execution_count": 8}, {"metadata": {"_uuid": "d826c140f6dd3cad0173226f6d9c3f4b13537926", "collapsed": true, "_cell_guid": "816e99a9-76a4-4ac5-99ec-12370368dbed"}, "outputs": [], "cell_type": "code", "source": ["# fifth convolutional layer\n", "W_conv5 = weight_variable([FILTER_SIZE, FILTER_SIZE, FILTER_NUM_4, FILTER_NUM_5], FILTER_NUM_4, FILTER_NUM_5)\n", "b_conv5 = bias_variable([FILTER_NUM_5], FILTER_NUM_4, FILTER_NUM_5)\n", "\n", "h_conv5 = lrelu(conv2d(h_pool4, W_conv5) + b_conv5)\n", "h_pool5 = h_conv5"], "execution_count": 9}, {"metadata": {"_uuid": "f19b12dd142820ee4ba2291660132ed5ca0734f7", "collapsed": true, "_cell_guid": "b4a78c52-c938-4e1c-952b-7869644e50ef"}, "outputs": [], "cell_type": "code", "source": ["# sixth convolutional layer\n", "W_conv6 = weight_variable([FILTER_SIZE, FILTER_SIZE, FILTER_NUM_5, FILTER_NUM_6], FILTER_NUM_5, FILTER_NUM_6)\n", "b_conv6 = bias_variable([FILTER_NUM_6], FILTER_NUM_5, FILTER_NUM_6)\n", "\n", "h_conv6 = lrelu(conv2d(h_pool5, W_conv6) + b_conv6)\n", "h_pool6 = max_pool_2x2(h_conv6)\n"], "execution_count": 10}, {"metadata": {"_uuid": "664ae7ad8c8b498fc1123528f367e2ffc4474847", "collapsed": true, "_cell_guid": "d9bb66d7-e200-4109-989a-2ac2eac2a14d"}, "outputs": [], "cell_type": "code", "source": ["# densely connected layer\n", "W_fc1 = weight_variable([int(image_width/8 * image_height/8 * FILTER_NUM_6), 1024], FILTER_NUM_6, 1024)\n", "b_fc1 = bias_variable([1024], FILTER_NUM_6, 1024)\n", "\n", "h_pool6_flat = tf.reshape(h_pool6, [-1, int(image_width/8 * image_height/8 * FILTER_NUM_6)])\n", "\n", "h_fc1 = lrelu(tf.matmul(h_pool6_flat, W_fc1) + b_fc1)"], "execution_count": 11}, {"metadata": {"_uuid": "e5eea54c264a1eb6f81521f6a5ef317b7e2ca697", "collapsed": true, "_cell_guid": "1b57614b-71ac-4cd0-ac91-f1ac1dd272cd"}, "outputs": [], "cell_type": "code", "source": ["# dropout\n", "keep_prob = tf.placeholder('float')\n", "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)"], "execution_count": 12}, {"metadata": {"_uuid": "2dfa7b837b37459b22a05723e7912f9dd0884df1", "collapsed": true, "_cell_guid": "f93a8a79-f926-4a0e-b710-ebb453133004"}, "outputs": [], "cell_type": "code", "source": ["# readout layer for deep net\n", "W_fc2 = weight_variable([1024, 2], 1024, 2)\n", "b_fc2 = bias_variable([2], 1024, 2)\n", "\n", "y = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)"], "execution_count": 13}, {"metadata": {"_uuid": "fc3e3abb1fc0c038b94d1b61f69d4be9df296f5f", "collapsed": true, "_cell_guid": "fc0230be-b6d3-4f86-af17-1a7abb4e8e99"}, "outputs": [], "cell_type": "code", "source": ["# cost function\n", "cross_entropy = -tf.reduce_sum(y_*tf.log(y))\n", "\n", "# optimisation function\n", "train_step = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(cross_entropy)\n", "\n", "# evaluation\n", "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))\n", "\n", "accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))"], "execution_count": 14}, {"metadata": {"_uuid": "f8777fe4a128804d736060f75bf2bbda356d132f", "collapsed": true, "_cell_guid": "6b39cb74-38e3-4f48-9d56-430a4deed7c2"}, "outputs": [], "cell_type": "code", "source": ["predict = tf.argmax(y,1)"], "execution_count": 15}, {"metadata": {"_uuid": "803f96327e75e3719ecbf52598f52fc657202c7a", "collapsed": true, "_cell_guid": "9b6c963c-8235-4d02-938e-c62c288f9b1d"}, "outputs": [], "cell_type": "code", "source": ["epochs_completed = 0\n", "index_in_epoch = 0\n", "num_examples = train_images.shape[0]\n", "\n", "# serve data by batches\n", "def next_batch(batch_size):\n", "    \n", "    global train_images\n", "    global train_labels\n", "    global index_in_epoch\n", "    global epochs_completed\n", "    \n", "    start = index_in_epoch\n", "    index_in_epoch += batch_size\n", "    \n", "    # when all training data have been already used, it is reordered randomly    \n", "    if index_in_epoch > num_examples:\n", "        # finished epoch\n", "        epochs_completed += 1\n", "        # shuffle the data\n", "        perm = np.arange(num_examples)\n", "        np.random.shuffle(perm)\n", "        train_images = train_images[perm]\n", "        train_labels = train_labels[perm]\n", "        # start next epoch\n", "        start = 0\n", "        index_in_epoch = batch_size\n", "        assert batch_size <= num_examples\n", "    end = index_in_epoch\n", "    return train_images[start:end], train_labels[start:end]"], "execution_count": 16}, {"metadata": {"_uuid": "908a75ce5a57f8d7e9a76cd616b44c47550fae7c", "collapsed": true, "_cell_guid": "bfe278c0-f832-4cb1-ad22-f62db1430ee5"}, "outputs": [], "cell_type": "code", "source": ["# start TensorFlow session\n", "init = tf.global_variables_initializer()\n", "sess = tf.InteractiveSession()\n", "\n", "sess.run(init)"], "execution_count": null}, {"metadata": {"_uuid": "6e89c03c65955926ec580731d58fab492829b239", "_cell_guid": "78fdfce5-8f13-42e7-8212-9fc5428d6e45"}, "outputs": [], "cell_type": "code", "source": ["# visualisation variables\n", "train_accuracies = []\n", "validation_accuracies = []\n", "x_range = []\n", "\n", "display_step=1\n", "\n", "for i in range(TRAINING_ITERATIONS):\n", "\n", "    #get new batch\n", "    batch_xs, batch_ys = next_batch(BATCH_SIZE)        \n", "    # check progress on every 1st,2nd,...,10th,20th,...,100th... step\n", "    if i%display_step == 0 or (i+1) == TRAINING_ITERATIONS:\n", "        \n", "        train_accuracy = accuracy.eval(feed_dict={x:batch_xs, \n", "                                                  y_:batch_ys, \n", "                                                  keep_prob: 1.0})\n", "        if(VALIDATION_SIZE):\n", "            validation_accuracy = accuracy.eval(feed_dict={ x: validation_images[0:BATCH_SIZE], \n", "                                                            y_: validation_labels[0:BATCH_SIZE], \n", "                                                            keep_prob: 1.0})                                  \n", "            print('training_accuracy / validation_accuracy / epoch=> %.2f / %.2f / %i for step %d'%(train_accuracy, validation_accuracy, epochs_completed, i))\n", "            \n", "            validation_accuracies.append(validation_accuracy)\n", "            \n", "        else:\n", "             print('training_accuracy => %.4f for step %d'%(train_accuracy, i))\n", "        train_accuracies.append(train_accuracy)\n", "        x_range.append(i)\n", "        \n", "        # increase display_step, max 1000\n", "        if i%(display_step*10) == 0 and i and display_step < 1000:\n", "            display_step *= 10\n", "    # train on batch\n", "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys, keep_prob: DROPOUT})\n", "    "], "execution_count": null}, {"metadata": {"_uuid": "0e47721e8f23eb99f50b772ab09d6247a05d3e76", "collapsed": true, "_cell_guid": "8e954cd4-2fed-40f7-934b-f4075dac3324"}, "outputs": [], "cell_type": "code", "source": [], "execution_count": null}], "nbformat": 4, "metadata": {"kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}}, "nbformat_minor": 1}