{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"def warn(*args, **kwargs):\n    pass\nimport warnings\nwarnings.warn = warn\n\nimport os\nprint(os.listdir(\"../input\"))\n\nimport glob\nimport numpy as np\nfrom skimage import io, transform\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport cv2\nimport time\nfrom random import shuffle\nimport tensorflow as tf\nfrom sklearn.metrics import confusion_matrix\nimport math\nfrom datetime import timedelta\nimport tensorflow.contrib.slim as slim\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87638ef3d7e958c97f851e033982ef220972c2b4"},"cell_type":"markdown","source":"# DATA PREPROCESSING"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"ROWS = 150\nCOLS = 150\nCHANNELS = 3\n\nBATCH_SIZE = 64\nLR_BASE = 1e-3\nnum_classes = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1771941d6f1a5aa86e9e26e0abfbd0d5e1ed2add"},"cell_type":"code","source":"train = glob.glob('../input/dogs-vs-cats-redux-kernels-edition/train/*.jpg')\nNUM_TRAINS = len(train)\nsplit_factor = 0.1\n\n# Shuffle training data\nshuffle(train)\n\nNUM_TRAININGS = int(NUM_TRAINS*(1 - split_factor))\n\nTRAINING_DATA = train[:NUM_TRAININGS]\n\nVALIDATION_DATA = train[NUM_TRAININGS:]\n\n(len(TRAINING_DATA), len(VALIDATION_DATA))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e60b0d0c3ed011ed22e3560e2d57c4580d4dfcf"},"cell_type":"code","source":"# Reshape color images\ndef reshaped_image(image):\n    return transform.resize(image,(ROWS, COLS, CHANNELS))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04e71318e9a7e11d76d5ede41ce3e06c6b66e0ff"},"cell_type":"code","source":"# One-hot Encoding for training data: cat [1 0], dog [0 1]\ndef process_Data(DIR_PATH):\n    path_Imgs = []\n    labels = []\n    for pathImg in DIR_PATH:\n        path_Imgs.append(pathImg)\n        imgName = pathImg.split('/')[-1]\n        label = imgName.split('.')[0]\n        if label == 'cat':\n            label = [1., 0.]\n        else:\n            label = [0., 1.]\n        labels.append(label)\n    \n    return np.array(path_Imgs), np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f5da6caa6a2695010935ed43e7da44408b193cc"},"cell_type":"code","source":"def loadBatchImages(BatchDataPath):\n    BatchImages = []\n    for path in BatchDataPath:\n        img = cv2.imread(path)\n\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = img/ 255.\n        BatchImages.append(reshaped_image(img))\n    return np.array(BatchImages)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f041d31c096349d62b813672a5252c4b6aed1d4"},"cell_type":"code","source":"TRAIN_PATH_DATAS, TRAIN_LABELS = process_Data(TRAINING_DATA)\nVAL_PATH_DATAS, VALID_LABELS = process_Data(VALIDATION_DATA)\nTRAIN_PATH_DATAS.shape, VAL_PATH_DATAS.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f95ce24ab37b528adaec027ae349b79c904817c1"},"cell_type":"code","source":"def loading_Data(DataPath, DataLabels, Num_Datas, current_index, batch_size=64):\n    begin_index = current_index*batch_size\n    end_index = -1\n    if begin_index + batch_size < Num_Datas:\n        end_index = begin_index + batch_size\n    else:\n        end_index = Num_Datas\n    Batch_Imgs = loadBatchImages(DataPath[begin_index : end_index])\n    Batch_Labels = DataLabels[begin_index : end_index]\n    \n    return Batch_Imgs, Batch_Labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17ead31472850bdcd3335b9410b1d3a27aabb26b"},"cell_type":"code","source":"Batch_Imgs, Batch_Labels = loading_Data(TRAIN_PATH_DATAS, TRAIN_LABELS, NUM_TRAININGS, current_index = 0, batch_size=9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"faf270796f27b4e6b811a67ff94ce4ea3865b9d6"},"cell_type":"code","source":"Batch_Imgs.shape, Batch_Labels.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58b05e08adbbe57776d283080096b50b86578aae"},"cell_type":"code","source":"def plot_image(images, cls_true, cls_pred=None):\n    assert len(images) == len(cls_true) == 9\n    fig, axes = plt.subplots(3, 3)\n    fig.subplots_adjust(hspace=0.3, wspace=0.3)\n    for i, ax in enumerate(axes.flat):\n        ax.imshow(images[i])\n    \n        if cls_pred is None:\n            xlabel = \"True: {0}\".format(cls_true[i])\n        else:\n            xlabel = \"True: {0}, Pred: {1}\".format(cls_true[i], cls_pred[i])\n    \n        ax.set_xlabel(xlabel)\n    \n        ax.set_xticks([])\n        ax.set_yticks([])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f64b5e1e009ec07fc9f29abec3152e0cf5a0603"},"cell_type":"code","source":"Batch_Labels_cls = np.argmax(Batch_Labels, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c1d118c39a1d6d388c9a4549f865b8ce8cab07c"},"cell_type":"code","source":"plot_image(Batch_Imgs, Batch_Labels_cls)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc4dee244caac46c287fe4934f6c21963eb6d55f"},"cell_type":"markdown","source":"# CONVOLUTION NEURAL NETWORK WITH TENSORFLOW"},{"metadata":{"_uuid":"567fc20c96c174d518eada8c4b1b6c227896406a"},"cell_type":"markdown","source":"## Load pre-trained VGG16 Model"},{"metadata":{"trusted":true,"_uuid":"13086b84ab881666cb898e399a8c816b1c025a23"},"cell_type":"code","source":"VGG_Weights = np.load('../input/vgg16-pretrained/vgg16.npy', encoding='latin1').item()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e3cf2000990f009649fb1ffbfb8df8361f06410"},"cell_type":"markdown","source":"## Placeholder variables"},{"metadata":{"trusted":true,"_uuid":"d065ce99175cfeaf5bc9455e10f28df3a69d1535"},"cell_type":"code","source":"x_image = tf.placeholder(tf.float32, shape=[None, ROWS, COLS, CHANNELS], name='x_image')\ny_true = tf.placeholder(tf.float32, shape=[None, num_classes], name='y_true')\ny_true_cls = tf.argmax(y_true, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe85c1e9175080d875b2c2fcbe3b52676c483b9a"},"cell_type":"markdown","source":"## Layers Implementation"},{"metadata":{"trusted":true,"_uuid":"5aca7d0306025faae98245ec0e54ea00dbca7fcc"},"cell_type":"code","source":"def conv2d(layer, name, n_filters, trainable, k_size=3):\n        return tf.layers.conv2d(layer, n_filters, kernel_size=(k_size, k_size),\n                                activation=tf.nn.relu, padding='SAME', name=name, trainable=trainable,\n                                kernel_initializer=tf.constant_initializer(VGG_Weights[name][0], dtype=tf.float32),\n                                bias_initializer=tf.constant_initializer(VGG_Weights[name][1], dtype=tf.float32),\n                                use_bias=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70f46e58827c8dbd20c748fafe46631d94eb1fe3"},"cell_type":"code","source":"net = x_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a27a1010cbc839710f495d5a4cb29a8ecb0b413b"},"cell_type":"code","source":"# Block 1\nnet = conv2d(net, 'conv1_1', 64, trainable = False)\nlayer_conv1_1 = net\nnet = conv2d(net, 'conv1_2', 64, trainable = False)\nlayer_conv1_2 = net\nnet = tf.layers.max_pooling2d(net, (2, 2), (2, 2), padding='same')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9451a2fa3545ff84db76d84d960db43d465bcbc6"},"cell_type":"code","source":"# Block 2\nnet = conv2d(net, 'conv2_1', 128, trainable = False)\nlayer_conv2_1 = net\nnet = conv2d(net, 'conv2_2', 128, trainable = False)\nlayer_conv2_2 = net\nnet = tf.layers.max_pooling2d(net, (2, 2), (2, 2), padding='same')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76f54f4dba0b95105e3f3eb7f9c22fe0eae2aeb8"},"cell_type":"code","source":"# Block 3\nnet = conv2d(net, 'conv3_1', 256, trainable = False)\nlayer_conv3_1 = net\nnet = conv2d(net, 'conv3_2', 256, trainable = False)\nlayer_conv3_2 = net\nnet = conv2d(net, 'conv3_3', 256, trainable = False)\nlayer_conv3_3 = net\nnet = tf.layers.max_pooling2d(net, (2, 2), (2, 2), padding='same')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12eb45d28d0c3a2bf6cc4e52989635bba6d513b2"},"cell_type":"code","source":"# Block 4\nnet = conv2d(net, 'conv4_1', 512, trainable = True)\nlayer_conv4_1 = net\nnet = conv2d(net, 'conv4_2', 512, trainable = True)\nlayer_conv4_2 = net\nnet = conv2d(net, 'conv4_3', 512, trainable = True)\nlayer_conv4_3 = net\nnet = tf.layers.max_pooling2d(net, (2, 2), (2, 2), padding='same')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2b67d29badce717d1d20da2230890386c42f2a3"},"cell_type":"code","source":"# Block 5\nnet = conv2d(net, 'conv5_1', 512, trainable = True)\nlayer_conv5_1 = net\nnet = conv2d(net, 'conv5_2', 512, trainable = True)\nlayer_conv5_2 = net\nnet = conv2d(net, 'conv5_3', 512, trainable = True)\nlayer_conv5_3 = net\nnet = tf.layers.max_pooling2d(net, (2, 2), (2, 2), padding='same')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f99aec8ba3ce19455c6cf84d97169645eba3c69"},"cell_type":"code","source":"layer_conv5_3, net","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"374683d5719bc41c02521fecb8e46f87aff9d7e0"},"cell_type":"code","source":"net = tf.contrib.layers.flatten(net)\nnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a7ece6b8cfc7021646b5086d67d610f336fd83f"},"cell_type":"code","source":"net = tf.layers.dense(inputs=net, \n                      name='layer_fc1',\n                      units=512, \n                      activation=tf.nn.relu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66331ba27202546e216286793ae70947b0ad97d4"},"cell_type":"code","source":"net = tf.layers.dense(inputs=net, \n                      name='layer_fc2',\n                      units=64, \n                      activation=tf.nn.relu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b29e94304b4518b35fed0b665ce90b602345763"},"cell_type":"code","source":"net = tf.layers.dense(inputs=net, \n                      name='layer_fc_out',\n                      units=num_classes, \n                      activation=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3203be98c6d1381f8dae215771d67630a5f784a1"},"cell_type":"code","source":"logits = net\ny_pred = tf.nn.softmax(logits=logits)\ny_pred_cls = tf.argmax(y_pred, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06fc6d910cca9e0496f55ec9ceb3a64ce6e8c026"},"cell_type":"code","source":"y_pred_cls","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"badfc512597a5e578283683ef93fb6252b59496a"},"cell_type":"markdown","source":"# Model Summary"},{"metadata":{"trusted":true,"_uuid":"b893b36e8016928f2a523bbb66e4006eae45cc0e"},"cell_type":"code","source":"def model_summary():\n    model_vars = tf.global_variables()\n    slim.model_analyzer.analyze_vars(model_vars, print_info=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56d1de4af8c7da832f00683e616c2387c376f430"},"cell_type":"code","source":"model_summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d1f902710c4d38417253dafb0cf82a6d3de5bd7"},"cell_type":"code","source":"def model_trainable_summary():\n    model_vars = tf.trainable_variables()\n    slim.model_analyzer.analyze_vars(model_vars, print_info=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7fa244b3a6a2037d755e866988a3f1f46711814"},"cell_type":"code","source":"# summary trainable weights\nmodel_trainable_summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec32db442f52fbf366c263b29fd9d22011ba5a98"},"cell_type":"markdown","source":"## Loss-Function to be Optimized"},{"metadata":{"trusted":true,"_uuid":"d73174beb96ae476b5b48d99fbe1c35b96f45e2a"},"cell_type":"code","source":"cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=logits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c122e88a1318363ca93b33de7065aa8dc2fc19d"},"cell_type":"code","source":"loss = tf.reduce_mean(cross_entropy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a68009d1c84f9cf7c8cd2a50a79757ffdc129cb"},"cell_type":"code","source":"# Optimization Method\noptimizer = tf.train.AdamOptimizer(learning_rate=1e-3).minimize(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5140b4f3fe26adaa3698551c8832ece04e06366"},"cell_type":"code","source":"# Classification Accuracy\ncorrect_prediction = tf.equal(y_pred_cls, y_true_cls)\naccuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4cbdf28a48698eecb1622f598b9aebb9089a5f3"},"cell_type":"markdown","source":"# Show Global Variables"},{"metadata":{"trusted":true,"_uuid":"8a08715ea38db1c9dd59983315c8a4f1e0492b0a"},"cell_type":"code","source":"for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):\n    print(var)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91f0ad241a18d4a1c0b1c3f0043ed45f33d71f58"},"cell_type":"code","source":"def get_weights_variable(layer_name):\n    with tf.variable_scope(layer_name, reuse=True):\n        variable = tf.get_variable('kernel')\n    return variable","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ceea5e25401721a1ef58dd753461db7a14ddc8ed"},"cell_type":"code","source":"weights_conv1_1 = get_weights_variable(layer_name='conv1_1')\nweights_conv1_2 = get_weights_variable(layer_name='conv1_2')\n\nweights_conv2_1 = get_weights_variable(layer_name='conv2_1')\nweights_conv2_2 = get_weights_variable(layer_name='conv2_2')\n\nweights_conv3_1 = get_weights_variable(layer_name='conv3_1')\nweights_conv3_2 = get_weights_variable(layer_name='conv3_2')\nweights_conv3_3 = get_weights_variable(layer_name='conv3_3')\n\nweights_conv4_1 = get_weights_variable(layer_name='conv4_1')\nweights_conv4_2 = get_weights_variable(layer_name='conv4_2')\nweights_conv4_3 = get_weights_variable(layer_name='conv4_3')\n\nweights_conv5_1 = get_weights_variable(layer_name='conv5_1')\nweights_conv5_2 = get_weights_variable(layer_name='conv5_2')\nweights_conv5_3 = get_weights_variable(layer_name='conv5_3')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a54a25ebd2f6d4167830296fd23b51cef02aa7bf"},"cell_type":"markdown","source":"# TensorFlow Run"},{"metadata":{"trusted":true,"_uuid":"ac8cf3fea9dd1819667b5c1b821eb27f860b3a9f"},"cell_type":"code","source":"session = tf.Session()\ninit = tf.global_variables_initializer()\nsession.run(init)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"369339f6e6b7b4101ddf113700c7cc30571d8f0b"},"cell_type":"code","source":"test_batch_size = 128\n\ndef print_test_accuracy(show_example_error=False, show_confusion_matrix=False):\n    num_test = len(VALIDATION_DATA)\n    cls_pred = np.zeros(shape=num_test, dtype=np.int)\n    i = 0\n    while i < num_test:\n        j = min(i + test_batch_size, num_test)\n\n        k = int(i // test_batch_size)\n\n        x_batch, y_true_batch = loading_Data(VAL_PATH_DATAS, \n                                           VALID_LABELS, \n                                           len(VALIDATION_DATA), \n                                           current_index = k, \n                                           batch_size=test_batch_size)\n\n        feed_dict = {x_image: x_batch, y_true: y_true_batch}\n\n        cls_pred[i:j] = session.run(y_pred_cls, feed_dict=feed_dict)\n\n        i = j\n    cls_true = np.argmax(VALID_LABELS, axis=1)\n    correct = (cls_true == cls_pred)\n    correct_sum = correct.sum()\n    acc = float(correct_sum) / num_test\n    msg = \"Accuracy on Validation-Set: {0:.1%} ({1} / {2})\"\n    print(msg.format(acc, correct_sum, num_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a5476514a1f51f46252cf58392e234150291bac"},"cell_type":"code","source":"train_batch_size = 128\ntotal_iterations = 0\n\ndef optimize(num_iterations):\n    global total_iterations\n    start_time = time.time()\n    num_batchs = math.ceil(NUM_TRAININGS / train_batch_size)\n    for epoch in range(total_iterations, num_iterations + total_iterations):\n    \n        for batch_step in range(num_batchs):\n            x_batch, y_true_batch = loading_Data(TRAIN_PATH_DATAS, \n                                               TRAIN_LABELS, \n                                               NUM_TRAININGS, \n                                               current_index = batch_step, \n                                               batch_size=train_batch_size)\n\n            feed_dict_train = {x_image: x_batch, y_true: y_true_batch}\n            session.run(optimizer, feed_dict = feed_dict_train)\n            if batch_step % 10 == 0:\n                train_acc = session.run(accuracy, feed_dict=feed_dict_train)\n                msg = \"Optimization Iteration: {0:>6}, Training Accuracy : {1:>6.1%}\"\n                print(msg.format(batch_step + 1, train_acc))\n        print('===============================')\n        print('Epoch : {}'.format(epoch))\n        print_test_accuracy()\n        print('===============================')\n    total_iterations += num_iterations\n    end_time = time.time()\n    time_dif = end_time - start_time\n    print(\"Time usage: \" + str(timedelta(seconds=int(round(time_dif)))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e283a1ed487ed60a2f0a68f5cb13465ce03ecbc8"},"cell_type":"code","source":"#print_test_accuracy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"daee4639a523197d9a6e4c17f6c59a2a6791734b"},"cell_type":"code","source":"optimize(num_iterations = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fe874503fe077c74fcc386fd681e6f14afe2bed"},"cell_type":"code","source":"#optimize(num_iterations = 9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccaa8ba1e64e07062e81f225e6eda50d4840b165"},"cell_type":"code","source":"#print_test_accuracy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88b063df895c6c19dde636c4b001940750f530b6"},"cell_type":"code","source":"#optimize(num_iterations = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63b8cc94844066e39a80525e0ea9fd927c435932"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}