{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b06a0476-c727-6e12-1b24-193ceb4a424b"},"outputs":[],"source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport os\nfrom IPython.display import display, Image, HTML\nimport tensorflow as tf\nimport cv2\nfrom time import strftime\n\nTRAIN_DIR = '../input/train/'\nTEST_DIR = '../input/test/'"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9ba0b0b7-43e7-befd-5d8d-19974ba6b0b7"},"outputs":[],"source":"# used for scaling/normalization\nIMAGE_SIZE = 96; # 150x150.  Also, 224, 96, 64, and 32 are also common\nCHANNELS = 3\npixel_depth = 255.0  # Number of levels per pixel.\n\nTRAINING_AND_VALIDATION_SIZE_DOGS = 1000 \nTRAINING_AND_VALIDATION_SIZE_CATS = 1000 \nTRAINING_AND_VALIDATION_SIZE_ALL  = 2000\nTRAINING_SIZE = 1600  # TRAINING_SIZE + VALID_SIZE must equal TRAINING_AND_VALIDATION_SIZE_ALL\nVALID_SIZE = 400\nTEST_SIZE_ALL = 500\n\nif (TRAINING_SIZE + VALID_SIZE != TRAINING_AND_VALIDATION_SIZE_ALL):\n    print (\"Error, check that TRAINING_SIZE+VALID_SIZE is equal to TRAINING_AND_VALIDATION_SIZE_ALL\")\n    exit ()\n\ntrain_images_path = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)] \ntrain_dogs =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'dog' in i]\ntrain_cats =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'cat' in i]\ntest_images_path =  [TEST_DIR+i for i in os.listdir(TEST_DIR)]\n\ntrain_images_path = train_dogs[:TRAINING_AND_VALIDATION_SIZE_DOGS] + train_cats[:TRAINING_AND_VALIDATION_SIZE_CATS]\ntrain_labels = np.array (([[1, 0]] * TRAINING_AND_VALIDATION_SIZE_DOGS) + ([[0, 1]] * TRAINING_AND_VALIDATION_SIZE_CATS))\ntest_images_path =  test_images_path[:TEST_SIZE_ALL]\n\ntrain_images_path = np.array(train_images_path)\ntest_images_path = np.array(test_images_path)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"6f3c6d75-d929-8ef7-277e-d21998c931fa"},"outputs":[],"source":"# resizes to IMAGE_SIZE/IMAGE_SIZE while keeping aspect ratio the same.  pads on right/bottom as appropriate \ndef read_image(file_path):\n    img = cv2.imread(file_path, cv2.IMREAD_COLOR) #cv2.IMREAD_GRAYSCALE\n    if (img.shape[0] >= img.shape[1]): # height is greater than width\n        resizeto = (IMAGE_SIZE, int (round (IMAGE_SIZE * (float (img.shape[1])  / img.shape[0]))));\n    else:\n        resizeto = (int (round (IMAGE_SIZE * (float (img.shape[0])  / img.shape[1]))), IMAGE_SIZE);\n    \n    resized = cv2.resize(img, (resizeto[1], resizeto[0]), interpolation=cv2.INTER_CUBIC)\n    padded = cv2.copyMakeBorder(resized, 0, IMAGE_SIZE - resized.shape[0], 0, IMAGE_SIZE - resized.shape[1], cv2.BORDER_CONSTANT, 0)\n        \n    return padded\n\ndef randomize(dataset, labels):\n    assert len(dataset) == len(labels)\n    p = np.random.permutation(len(dataset))\n    shuffled_dataset = dataset[p]\n    shuffled_labels = labels[p]\n    \nrandomize(train_images_path, train_labels)\n\ntrain_images = np.array([read_image(path) for path in train_images_path])\ntest_images = np.array([read_image(path) for path in test_images_path])\n\nvalid_images = train_images[:VALID_SIZE]\nvalid_labels = train_labels[:VALID_SIZE]\ntrain_images = train_images[VALID_SIZE:VALID_SIZE+TRAINING_SIZE]\ntrain_labels  = train_labels[VALID_SIZE:VALID_SIZE+TRAINING_SIZE]\n\nplt.imshow(cv2.cvtColor(train_images[0], cv2.COLOR_BGR2RGB))\nprint(train_labels[0])"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d3cc0bbc-7100-4fb2-fe01-09b8f9ff4f02"},"outputs":[],"source":"def flatten_cnn(layer):\n    layer_shape = layer.get_shape().as_list()\n    n_out = layer_shape[1] * layer_shape[2] * layer_shape[3]\n    return tf.reshape(layer, [-1, n_out])\n\ndef build_nn(shape, X, name):\n    n_before = int(X.get_shape()[1])\n    W = tf.Variable(tf.truncated_normal([n_before, shape], stddev=0.1), name=name+\"_W\")\n    b = tf.Variable(tf.constant(0.1, shape=[shape]), name=name+\"_b\")\n    layer = tf.matmul(X, W)+b\n    return layer\n\ndef build_cnn(cnn_shape, patch_shape, X, name, stride=1):\n    n_before = int(X.get_shape()[3])\n    W = tf.Variable(tf.truncated_normal([patch_shape[0], patch_shape[1], n_before, cnn_shape], stddev=0.1),\n                   name=name+\"_W\")\n    b = tf.Variable(tf.constant(0.1, shape=[cnn_shape]), name=name+\"_b\")\n    layer = tf.nn.conv2d(X, W, strides=[1, stride, stride, 1], padding='SAME') + b\n    return layer\n\ndef build_cnn_relu(cnn_shape, patch_shape, X, name, stride=1):\n    layer = tf.nn.relu(build_cnn(cnn_shape, patch_shape, X, name, stride))\n    return layer\n\ndef max2d_pool(layer):\n    return tf.nn.max_pool(layer, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n\n# http://r2rt.com/implementing-batch-normalization-in-tensorflow.html\ndef batch_norm(layer, is_training, name, decay = 0.999, does_scale=True):\n    layer_shape = layer.get_shape().as_list()\n    depth_dim = len(layer_shape)-1\n    depth = layer_shape[-1]\n    if does_scale:\n        scale = tf.Variable(tf.ones(depth), name=name+\"_BN_s\")\n    else:\n        scale = None\n    beta = tf.Variable(tf.zeros(depth), name=name+\"_BN_b\")\n    mov_mean = tf.Variable(tf.zeros(depth), trainable=False, name=name+\"_BN_pm\")\n    mov_var = tf.Variable(tf.ones(depth), trainable=False, name=name+\"_BN_pv\")\n    \n    def use_batch_with_update_mov():\n        batch_mean, batch_var = tf.nn.moments(layer,list(range(depth_dim)))\n        train_mean = tf.assign(mov_mean,\n                               mov_mean * decay + batch_mean * (1 - decay))\n        train_var = tf.assign(mov_var,\n                              mov_var * decay + batch_var * (1 - decay))\n        with tf.control_dependencies([train_mean, train_var]):\n            return tf.nn.batch_normalization(layer,\n                batch_mean, batch_var, beta, scale, 0.001)\n        \n    def use_mov():\n        return tf.nn.batch_normalization(layer,\n            mov_mean, mov_var, beta, scale, 0.001)\n    \n    return tf.cond(is_training, use_batch_with_update_mov, use_mov)\n    \ndef build_nn_bn(shape, X, is_training, name, decay = 0.999, does_scale=True):\n    n_before = int(X.get_shape()[1])\n    W = tf.Variable(tf.truncated_normal([n_before, shape], stddev=0.1), name=name+\"_W\")\n    layer = tf.matmul(X, W)\n    return batch_norm(layer, is_training, name, decay, does_scale)\n\ndef build_nn_bn_relu(shape, X, is_training, name, decay = 0.999):\n    layer = tf.nn.relu(build_nn_bn(shape, X, is_training, name, decay, False))\n    return layer\n    \ndef build_cnn_bn(cnn_shape, patch_shape, X, is_training, name, stride=1, decay=0.999, does_scale=True):\n    n_before = int(X.get_shape()[3])\n    W = tf.Variable(tf.truncated_normal([patch_shape[0], patch_shape[1], n_before, cnn_shape], stddev=0.1),\n                   name=name+\"_W\")\n    layer = tf.nn.conv2d(X, W, strides=[1, stride, stride, 1], padding='SAME')\n    return batch_norm(layer, is_training, name, decay, does_scale)\n\ndef build_cnn_bn_relu(cnn_shape, patch_shape, X, is_training, name, stride=1, decay=0.999):\n    layer = tf.nn.relu(build_cnn_bn(cnn_shape, patch_shape, X, is_training, name, stride, decay, False))\n    return layer\n\ndef slice_label(tf_label, len_tuple):\n    cur = 0\n    sliced = []\n    for l in len_tuple:\n        sliced.append(tf.slice(tf_label, [0, cur], [-1, l]))\n        cur += l\n    return tuple(sliced)\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"df5ac803-319f-c64d-e91a-b05f5b4cd9b7"},"outputs":[],"source":"tf.reset_default_graph()\nX = tf.placeholder(tf.float32, [None, 96, 96, 3])\nis_training = tf.placeholder(tf.bool)\n\n# Small inception model\n# http://laonple.blog.me/220704822964\ncnn_1_5 = build_cnn_bn_relu(12, [5,5], X, is_training, \"cnn_1_5\")\ncnn_1_3 = build_cnn_bn_relu(36, [3,3], X, is_training, \"cnn_1_3\")\ncnn_1_concat = tf.concat([cnn_1_5, cnn_1_3], 3)\ncnn_1_pool = max2d_pool(cnn_1_concat) # 48 * 48 * 48\n\ncnn_2_5 = build_cnn_bn_relu(18, [5,5], cnn_1_pool, is_training, \"cnn_2_5\")\ncnn_2_3 = build_cnn_bn_relu(48, [3,3], cnn_1_pool, is_training, \"cnn_2_3\")\ncnn_2_1 = build_cnn_bn_relu(30, [1,1], cnn_1_pool, is_training, \"cnn_2_1\")\ncnn_2_concat = tf.concat([cnn_2_5, cnn_2_3, cnn_2_1], 3)\ncnn_2_pool = max2d_pool(cnn_2_concat) # 24 * 24 * 96\n\ncnn_3_3_reduce = build_cnn_bn_relu(32, [1,1], cnn_2_pool, is_training, \"cnn_3_3_reduce\")\ncnn_3_3 = build_cnn_bn_relu(48, [3,3], cnn_3_3_reduce, is_training, \"cnn_3_3\")\ncnn_3_1 = build_cnn_bn_relu(16, [1,1], cnn_2_pool, is_training, \"cnn_3_1\")\ncnn_3_concat = tf.concat([cnn_3_3, cnn_3_1], 3)\ncnn_3_pool = max2d_pool(cnn_3_concat) # 12 * 12 * 64\n\ndense_1 = build_nn_bn_relu(1024, flatten_cnn(cnn_3_pool), is_training, \"dense_1\")\n\nlogit = build_nn(2, dense_1, \"logit\")\nhyp = tf.nn.softmax(logit)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"1d954b5e-d00d-0f83-456a-6046a5657f90"},"outputs":[],"source":"Y = tf.placeholder(tf.float32, [None, 2])\n\nlearning_rate = tf.placeholder(tf.float32)\ncost = tf.nn.softmax_cross_entropy_with_logits(logits=logit, labels=Y)\ncost_mean = tf.reduce_mean(cost) # mean of batch set\n\ntrain = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n\ncorrect = tf.equal(tf.argmax(Y,1), tf.argmax(hyp,1))\naccuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n\ndef get_now_str():\n    return strftime(\"%Y-%m-%d %H:%M:%S\")\n\ndef do_train(max_epoch=4, batchsize=200, lr_init = 0.003, prog=False, stat=True):\n    sess = tf.Session()\n    sess.run(tf.initialize_all_variables())\n\n    trainsize = train_images.size\n    batch_per_epoch = int(trainsize/batchsize)\n    print (\"[%s] Training %d, mini-batch %d * %d\" % (get_now_str(), trainsize, batchsize, batch_per_epoch))\n    epoch = 0\n\n    i = 0\n    num_trained = 0\n    lr = lr_init\n    \n    train_index = 0\n    \n    while (epoch < max_epoch):\n        batch_x = train_images[train_index:batchsize+train_index]\n        batch_y = train_labels[train_index:batchsize+train_index]\n        train_index += batchsize\n\n        i += 1\n        num_trained += batch_x.shape[0]\n\n        cur_cost = sess.run((train, cost_mean),\n                            feed_dict={X:batch_x, Y:batch_y, is_training: True, learning_rate:lr})[1]\n\n        if i % 10 == 0 :\n            if prog == True:\n                print(\"                                        \\r\", end=\"\")\n            if stat == True :\n                cv_cost, cv_acc = get_mean_in_batch(sess, (cost_mean, accuracy), cvimg, cvlabel, executor)\n                cur_cost_test, cur_acc = sess.run((cost_mean, accuracy), feed_dict={X:batch_x, Y:batch_y, is_training: False})\n                print (\"[%s] %5.2f %4.2e %4.3f %4.2e %4.3f %3.2e\" %\n                    (get_now_str(), num_trained/trainsize, cur_cost_test, cur_acc, cv_cost, cv_acc, lr))\n            else :\n                print (\"[%s] %4.2f %4.2e\" % (get_now_str(), num_trained/trainsize, cur_cost))\n        if prog == True:\n            print (\"%dth... lr = %.2e, cost = %.2e\\r\" % (i, lr, cur_cost), end=\"\")\n        lr = lr * (1 - 0.0003)\n    print(\"                                        \\r\", end=\"\")\n    print(\"[%s] train complete\" % get_now_str())\n    print(\"test accuracy ---\")\n    print_accuracy(sess, testimg, testlabel, True, executor)\n    print(\"train accuracy ---\")\n    print_accuracy(sess, trainimg, trainlabel, True, executor)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7bbdaca3-8a5c-ca83-5c57-1399ca55e9e9"},"outputs":[],"source":"do_train()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f4f9dbc3-69e8-3665-46a3-6ad11a0c62e8"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}