{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\n\nimport math\nimport time\n\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"310a92c40e1211fb2a04b7edfcd755ef14b8f7cb"},"cell_type":"code","source":"# I impletemented with Keras and Tensorflow, in order to learn their difference\n# True - run in Keras, False - run in Tensorflow\nrunInKeras = True","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"53717d8f-d15a-4ae1-9688-8b8c2b61dafe","collapsed":true,"_uuid":"4fe19461bd9045dadf9fb0605f780d0ed769c79d","trusted":true},"cell_type":"code","source":"CLASSES = [\"cat\", \"dog\"]\n\nIMG_SHAPE = (80, 80, 3)   # (height, width, channels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b16323c0259b3746aee0fee0aca402472b28445b"},"cell_type":"code","source":"from keras.preprocessing import image\n\ndef load_image(img_dir, img_name, img_size):\n    \"\"\"\n    img_size: (height, width)\n    \"\"\"\n    \n    img_path = img_dir + img_name\n    x = image.load_img(img_path, target_size = img_size)\n    x = image.img_to_array(x)\n    x = x / 255\n#     x = np.expand_dims(x, axis=0)   # For channel = 1\n    return x","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"73f27305-36be-41a7-a897-5f18dac67e2f","_uuid":"d5997d3571766db7f793d0a90bb7114aa4f9964d","trusted":true,"collapsed":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom keras.applications.imagenet_utils import preprocess_input\n\ndef load_dataset(img_dir, img_names=None, test_size=None, print_progress=False):\n    if img_names is None:\n        img_names = os.listdir(img_dir)\n        \n    m = len(img_names)\n    X = np.empty((m, IMG_SHAPE[0], IMG_SHAPE[1], IMG_SHAPE[2]), dtype=np.float32)\n    Y = np.zeros((m, 1))\n    \n    for i, img_name in enumerate(img_names):\n        X[i] = load_image(img_dir, img_name, (IMG_SHAPE[0], IMG_SHAPE[1]))\n#         X[i] = preprocess_input(load_image(img_dir, img_name, (IMG_SHAPE[0], IMG_SHAPE[1]))   # For ResNet\n        \n        if 'dog' in img_name:\n            Y[i] = 1\n        \n        if print_progress and i % 100 == 0:\n            print('Processed {0} of {1}'.format(i, m), end='\\r')\n            \n    if test_size is None:\n        return X, Y\n    else:\n        return train_test_split(X, Y, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"44e9942b232a603a1e65fdf6bfc04afe4d01d86f"},"cell_type":"code","source":"def plot_train_history(history):\n    # plot the cost and accuracy \n    loss_list = history['loss']\n    val_loss_list = history['val_loss']\n    accuracy_list = history['acc']\n    val_accuracy_list = history['val_acc']\n    # epochs = range(len(loss_list))\n\n    # plot the cost\n    plt.plot(loss_list, 'b', label='Training cost')\n    plt.plot(val_loss_list, 'r', label='Validation cost')\n    plt.ylabel('cost')\n    plt.xlabel('iterations')\n    plt.title('Training and validation cost')\n    plt.legend()\n    \n    plt.figure()\n    \n    # plot the accuracy\n    plt.plot(accuracy_list, 'b', label='Training accuracy')\n    plt.plot(val_accuracy_list, 'r', label='Validation accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('iterations')\n    plt.title('Training and validation accuracy')\n    plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a887783061a24aacea8d3a1b968769345803e20"},"cell_type":"markdown","source":"**Load all data**"},{"metadata":{"trusted":true,"_uuid":"100253d7a071cc0b62c4b6720b167a73bbd7a14d"},"cell_type":"code","source":"X_train, X_test, Y_train, Y_test = load_dataset('../input/train/', test_size=0.1, print_progress=True)\nprint(\"Train shape: {}\".format(X_train.shape))\nprint(\"Test shape: {}\".format(X_test.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d3750530e5903db4914b75905e064a60628c97d"},"cell_type":"code","source":"def plot_image(image, label, classes=None):\n    img_shape = image.shape   # height, width, channels\n    \n    if classes is None:\n        title = label\n    else:\n        if not np.isscalar(label):\n            label = np.argmax(label)\n        title = classes[int(label)]\n        \n    plt.figure(figsize=(6, 4))\n    plt.imshow(np.squeeze(image.reshape(img_shape[0], img_shape[1], img_shape[2])), interpolation='nearest')\n    plt.title(title)\n\n\n\nif runInKeras:\n    image_id = 0\n    plot_image(X_test[image_id, :], Y_test[image_id], CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64e1b9e50b2fb11d8818c69e0a23016b25e281a8"},"cell_type":"markdown","source":"**Only load image names**\n\nFor kernal memory limitation using in random_mini_batches"},{"metadata":{"_cell_guid":"2ca73eed-4eb2-4861-a73b-594866106169","_uuid":"607bf58982fb2842c4cf28797b3e327b7e45f2f4","trusted":true,"collapsed":true},"cell_type":"code","source":"def load_img_names(img_dir, test_size=0.1):\n    img_names = os.listdir(img_dir)\n    m = len(img_names)\n    spliter = int(m * (1 - test_size))\n    \n    permutation = np.random.permutation(m)\n    shuffled_img = np.array(img_names)[permutation]\n    img_train, img_test = shuffled_img[: spliter], shuffled_img[spliter :]\n    \n    return img_train, img_test\n\n\n\nif not runInKeras:\n    img_train, img_test = load_img_names('../input/train/')\n\n    X_test, Y_test = load_dataset('../input/train/', img_names = img_test, print_progress=True)\n    print(\"Test shape: {}\".format(X_test.shape))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4e8f0ba5-ef21-4d87-a6e0-a87efcb436ac","_uuid":"12076787fd5dfaf928dadce4d1f2eace0f9633fd"},"cell_type":"markdown","source":"**Keras**"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"bef24747520e4e7f29d2fa24b96149f3044fec0e"},"cell_type":"code","source":"from keras import backend as K\nfrom keras.layers import Input, Add, Multiply, Average, Maximum, Dense, Activation, ZeroPadding2D, BatchNormalization, Dropout, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D\nfrom keras.models import Model, load_model\nfrom keras.initializers import glorot_uniform, he_uniform\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.utils import to_categorical, layer_utils, plot_model\nfrom keras.utils.data_utils import get_file\nfrom keras.utils.vis_utils import model_to_dot\nfrom keras.applications.resnet50 import ResNet50","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"15ccbab7-5632-4f50-af16-f83d8d9418b8","collapsed":true,"_uuid":"66caf06f15b7a8ca60da0e6262b93df492fdfa42","trusted":false},"cell_type":"code","source":"def Res_block(X, filters, kernel_size, strides, name=\"Res\"):\n    F1, F2, F3 = filters\n    \n    # Save the input value\n    X_shortcut = X\n\n    ##### MAIN PATH #####\n    # First component of main path \n    X = Conv2D(F1, kernel_size, strides = strides, padding = 'valid', kernel_initializer = glorot_uniform(seed=0), name = \"{}_Conv1\".format(name))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n    \n    # Second component of main path\n    X = Conv2D(F2, kernel_size, strides = strides, padding = 'same', kernel_initializer = glorot_uniform(seed=0), name = \"{}_Conv2\".format(name))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n\n    # Third component of main path\n    X = Conv2D(F3, kernel_size, strides = strides, padding = 'valid', kernel_initializer = glorot_uniform(seed=0), name = \"{}_Conv3\".format(name))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2c')(X)\n\n    ##### SHORTCUT PATH ####\n    if tf.shape(X_shortcut)[3] != F3:\n        X_shortcut = Conv2D(F3, kernel_size, strides = strides, padding = 'valid', kernel_initializer = glorot_uniform(seed=0), name = \"{}_ShortCut\".format(name))(X_shortcut)\n        X_shortcut = BatchNormalization(axis = 3, name = bn_name_base + '1')(X_shortcut)\n\n    # Final step: Add shortcut value to main path, and pass it through a RELU activation\n    X = layers.Add()([X, X_shortcut])\n    X = Activation('relu')(X)\n    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e2476510-2159-416f-903d-ffaf968d5640","collapsed":true,"_uuid":"ac7c825d39f4b9fea727d446caab7a49a4f5247d","trusted":true},"cell_type":"code","source":"def MyModel(input_shape, num_classes=2):\n    # Define the input placeholder as a tensor with shape input_shape. Think of this as your input image!\n    X_input = Input(input_shape)\n\n    # Zero-Padding: pads the border of X_input with zeroes\n    X = ZeroPadding2D((3, 3))(X_input)\n\n    # CONV -> BN -> RELU Block applied to X\n    X = Conv2D(32, (3, 3), strides = (1, 1), padding='same', kernel_initializer=glorot_uniform(), name = 'Conv1')(X)\n#     X = BatchNormalization(axis = 3, name = 'BN1')(X)\n    X = Activation('relu')(X)\n    X = Dropout(0.5)(X)\n    \n    X = Conv2D(64, (3, 3), strides = (1, 1), padding='same', kernel_initializer=glorot_uniform(), name = 'Conv2')(X)\n#     X = BatchNormalization(axis = 3, name = 'BN2')(X)\n    X = Activation('relu')(X)\n    X = Dropout(0.5)(X)\n    \n    X = Conv2D(128, (3, 3), strides = (1, 1), padding='same', kernel_initializer=glorot_uniform(), name = 'Conv3')(X)\n#     X = BatchNormalization(axis = 3, name = 'BN3')(X)\n    X = Activation('relu')(X)\n    X = Dropout(0.5)(X)\n    \n    X = Conv2D(128, (3, 3), strides = (1, 1), padding='same', kernel_initializer=glorot_uniform(), name = 'Conv4')(X)\n#     X = BatchNormalization(axis = 3, name = 'BN4')(X)\n    X = Activation('relu')(X)\n    X = Dropout(0.5)(X)\n    \n    # MAXPOOL\n    X = MaxPooling2D((2, 2), name='MP')(X)\n\n\n#     Res_model = ResNet50(weights='imagenet', include_top=False, input_tensor=X_input)\n#     X = Res_model.output\n    \n    \n    # FLATTEN X (means convert it to a vector)\n    X = Flatten()(X)\n    \n    # FULLY CONNECTED\n    X = Dense(128, activation='relu', name='FC1')(X)\n    \n    if num_classes > 2:\n        X = Dense(num_classes, activation='softmax', name='FC2')(X)\n    else:\n        X = Dense(1, activation='sigmoid', name='FC2')(X)\n\n    # Create model. This creates your Keras model instance, you'll use this instance to train/test the model.\n    model = Model(inputs = X_input, outputs = X, name='CNN')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"34e335d4-9049-434a-ac5a-7a48d6fbb644","scrolled":true,"_uuid":"b376ef42f6fee6cecd6a7d8ad25e2521da1d7ab0","trusted":true,"collapsed":true},"cell_type":"code","source":"if runInKeras:\n    model = MyModel(IMG_SHAPE, len(CLASSES))\n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"726cb912f6106e4a5c4923acaea10d21acc2988a"},"cell_type":"code","source":"if runInKeras:\n    model.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.0003, decay=1e-6, beta_1=0.9, beta_2=0.999), metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"_uuid":"8eedc5d3d1a34f9384c03ed71e729bf96704db95","_kg_hide-output":true,"_cell_guid":"7726c982-915f-4a65-8618-fca7ef8ff596","_kg_hide-input":true,"trusted":true,"collapsed":true},"cell_type":"code","source":"if runInKeras:\n    monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=10, verbose=1, mode='auto')\n    checkpoint = ModelCheckpoint(filepath='best_weights.hdf5', verbose=1, save_best_only=True)   # Save the best model\n    hist = model.fit(X_train, Y_train, batch_size=128, callbacks=[monitor, checkpoint], epochs=50, shuffle=True, verbose=1, validation_split=0.01)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1dc17ff6-5ea7-46d8-ba10-6f82b728f6f8","_uuid":"84223d014f2229705d80e817bdc97e0edacc38b9","trusted":true,"collapsed":true},"cell_type":"code","source":"if runInKeras:\n    plot_train_history(hist.history)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"be033efb-2fd4-4981-a29a-5d95ccc267b4","_uuid":"72d785d6667069625307bdd095906cbecbda6b27","trusted":true,"collapsed":true},"cell_type":"code","source":"if runInKeras:\n    score = model.evaluate(X_test, Y_test)\n\n    print (\"Test Loss = \" + str(score[0]))\n    print (\"Test Accuracy = \" + str(score[1]))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6887c730-683c-4d5e-9884-9c236c1bba85","_uuid":"240446dafb77db5bf30b999981ac1d15aaef2b5e","trusted":true,"collapsed":true},"cell_type":"code","source":"if runInKeras:\n    Y_test_pred = model.predict(X_test, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3fcec06c-5a89-4db8-86b7-993e96f6fea1","_uuid":"7c9dcda9012c9576d1f4368e4c4c7f669fa1d6d4"},"cell_type":"markdown","source":"**Tensor Flow**\n\n1. create_placeholders function\n2. Create layers, including its scope and parameters\n3. compute_accuracy function\n4. compute_cost function\n5. forward_propagation function\n6. tf.reset_default_graph\n7. optimizer\n8. init, sess and run"},{"metadata":{"_cell_guid":"4edb831c-0002-403c-bf1a-c35394a16f50","collapsed":true,"_uuid":"60d934ca92029a0f10408aa1c11ad5f8183f1566","trusted":true},"cell_type":"code","source":"def create_placeholders(img_shape, n_y):\n    \"\"\"\n    Creates the placeholders for the tensorflow session.\n    \n    Arguments:\n    img_shape -- height, width, number of channels of an input image\n    n_y -- scalar, number of classes\n        \n    Returns:\n    X -- placeholder for the data input, of shape [None, n_H0, n_W0, n_C0] and dtype \"float\"\n    Y -- placeholder for the input labels, of shape [None, n_y] and dtype \"float\"\n    \"\"\"\n    \n    n_H0, n_W0, n_C0 = img_shape\n    \n    with tf.name_scope(\"Inputs\"):\n        X = tf.placeholder(tf.float32, shape=[None, n_H0, n_W0, n_C0])\n        Y = tf.placeholder(tf.float32, shape=[None, n_y])\n    \n    return X, Y","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8c7fbd7b-0625-4596-914d-a95c575fc7f5","collapsed":true,"_uuid":"677c2ca851289751536834f410746b8e7c6c6636","trusted":true},"cell_type":"code","source":"cache = {}\n\ndef Conv_Layer(input, filters, kernel_size, strides, keep_prob=1, name=\"Conv2D\", only_Conv=False, skip_MaxPool=False):\n    channel_in, channel_out = int(input.shape[3]), filters\n    filter_H, filter_W = kernel_size\n    stride_H, stride_W = strides\n    \n    with tf.name_scope(name):\n        W = tf.Variable(tf.truncated_normal([filter_H, filter_W, channel_in, channel_out], stddev=0.1), name = \"{}_W\".format(name))\n        cache[\"{}_W\".format(name)] = W\n        tf.summary.histogram(\"{}_W\".format(name), W)\n\n        conv = tf.nn.conv2d(input, W, strides=[1, stride_H, stride_W, 1], padding=\"SAME\")\n        if only_Conv:\n            return conv\n        \n#         tf.nn.batch_normalization(conv, )\n        b = tf.Variable(tf.constant(0.1, shape=[channel_out]), name=\"{}_b\".format(name))\n        tf.summary.histogram(\"{}_b\".format(name), b)\n        \n        A = tf.nn.relu(tf.add(conv, b))\n        cache[\"{}_A\".format(name)] = A\n        tf.summary.histogram(\"{}_A\".format(name), A)\n        \n        if keep_prob == 1:\n            D = A\n        else:\n            D = tf.nn.dropout(A, keep_prob)\n        \n        if skip_MaxPool:\n            return D\n        \n        MP = tf.nn.max_pool(D, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding=\"VALID\")\n        \n    return MP","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ecba55b4-4697-4420-a251-7358682d852e","collapsed":true,"_uuid":"777cffaba270694c4ea9479370c45dd659a4e5ea","trusted":true},"cell_type":"code","source":"def Dense_Layer(input, units, activation=None, name=\"Dense\"):\n    channel_in, channel_out = int(input.shape[1]), units\n\n    with tf.name_scope(name):\n        W = tf.Variable(tf.truncated_normal([channel_in, channel_out], stddev=0.1), name = \"{}_W\".format(name))\n        b = tf.Variable(tf.constant(0.1, shape=[channel_out]), name=\"{}_b\".format(name))\n    \n        Z = tf.matmul(input, W) + b\n    \n        tf.summary.histogram(\"{}_W\".format(name), W)\n        tf.summary.histogram(\"{}_b\".format(name), b)\n        tf.summary.histogram(\"{}_Z\".format(name), Z)\n        \n        if activation is None:\n            return Z\n        elif activation == \"relu\":\n            A = tf.nn.relu(Z)\n        elif activation == \"sigmoid\":\n            A = tf.nn.sigmoid(Z)\n        elif activation == \"softmax\":\n            A = tf.nn.softmax(Z)\n\n    return Z, A","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"20a02192-5680-482d-a713-9804eef59294","collapsed":true,"_uuid":"94bd44cc7b085707a86076bc2b179bb3ca3fe85c","trusted":true},"cell_type":"code","source":"def Res_block(X, filters, kernel_size, strides, name=\"Res\"):\n    F1, F2, F3 = filters\n    \n    # Save the input value\n    X_shortcut = X\n\n    ##### MAIN PATH #####\n    X = Conv_Layer(X, filters = F1, kernel_size = kernel_size, strides = strides, name = \"{}_Conv1\".format(name), skip_MaxPool=True)\n    X = Conv_Layer(X, filters = F2, kernel_size = kernel_size, strides = strides, name = \"{}_Conv2\".format(name), skip_MaxPool=True)\n    X = Conv_Layer(X, filters = F3, kernel_size = kernel_size, strides = strides, name = \"{}_Conv3\".format(name), only_Conv=True)\n    \n    ##### SHORTCUT PATH ####\n    if X_shortcut.shape[3] != F3:\n        X_shortcut = Conv_Layer(X, filters = F3, kernel_size = kernel_size, strides = (1, 1), name = \"{}_ShortCut\".format(name))\n\n    # Add shortcut value to main path, and pass it through a RELU activation\n    X = tf.nn.relu(tf.add(X, X_shortcut))\n    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"53373e23-4ef6-4fc9-b36a-6e3c843b00a1","collapsed":true,"_uuid":"58387a33690071578b9c1dc208854ed2f3ca4ce2","trusted":true},"cell_type":"code","source":"def compute_cost(Z, Y, activation=\"softmax\"):\n    if activation == \"sigmoid\":\n        cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = Z, labels = Y))\n    elif activation == \"softmax\":\n        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = Z, labels = Y))\n    \n    return cost","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0c93d71b-828b-4179-9bd4-34693612528d","collapsed":true,"_uuid":"1c8209eab44dc366a896e83777d1556589082b37","trusted":true},"cell_type":"code","source":"def compute_accuracy(Y_pred, Y, activation=\"softmax\"):\n    if activation == \"sigmoid\":\n#         correct_prediction = tf.equal(Y_pred > 0.5, tf.cast(Y, tf.bool))\n#         correct_prediction = tf.equal(tf.to_float(Y_pred > 0.5), Y)\n        correct_prediction = tf.equal(tf.round(Y_pred), Y)\n    elif activation == \"softmax\":\n        correct_prediction = tf.equal(tf.argmax(Y_pred, 1), tf.argmax(Y, 1))\n    \n    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n    \n    return accuracy","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"453c6dc8-0a69-46c1-ac50-e7d988ebbaae","collapsed":true,"_uuid":"2e6df82fc768718da2f2167b28ede9d592ebf67e","trusted":true},"cell_type":"code","source":"def forward_propagation(X, num_classes=2):\n    \"\"\"\n    Implements the forward propagation for the model\n    \n    Arguments:\n    X -- input dataset placeholder, of shape (input size, number of examples)\n\n    Returns:\n    Z -- the output of the last LINEAR unit\n    A -- the output of the last ACTIVATION unit\n    \"\"\"\n    \n    X = Conv_Layer(X, filters = 32, kernel_size = (3, 3), strides = (1, 1), keep_prob = 0.5, name = \"Conv1\")\n    X = Conv_Layer(X, filters = 64, kernel_size = (3, 3), strides = (1, 1), keep_prob = 0.5, name = \"Conv2\")\n    X = Conv_Layer(X, filters = 128, kernel_size = (3, 3), strides = (1, 1), keep_prob = 0.5, name = \"Conv3\")\n    X = Conv_Layer(X, filters = 128, kernel_size = (3, 3), strides = (1, 1), keep_prob = 0.5, name = \"Conv4\")\n    \n#     X = Res_block(X, filters = (32, 32, 64), kernel_size = (3, 3), strides = (1, 1), name = \"Res1\")\n#     X = Res_block(X, filters = (32, 32, 64), kernel_size = (3, 3), strides = (1, 1), name = \"Res2\")\n#     X = Res_block(X, filters = (32, 32, 64), kernel_size = (3, 3), strides = (1, 1), name = \"Res3\")\n    \n    # FLATTEN\n    X = tf.contrib.layers.flatten(X)\n    \n    # FULLY CONNECTED\n    _, X = Dense_Layer(X, 128, \"relu\", name = \"FC1\")\n    \n    if num_classes > 2:\n        Z, A = Dense_Layer(X, 1, \"softmax\", name = \"FC2\")\n    else:\n        #     Z = tf.contrib.layers.fully_connected(F2, 1, activation_fn=None)\n        Z, A = Dense_Layer(X, 1, \"sigmoid\", name = \"FC2\")\n\n    return Z, A","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b6dea5f7e39a0b34aa1aa01d7c3ebca7ebbadb34"},"cell_type":"code","source":"def random_mini_batches(X, Y, mini_batch_size = 64):\n# def random_mini_batches(X, mini_batch_size = 64):\n    m = X.shape[0]                  # number of training examples\n    mini_batches = []\n    \n    # Step 1: Shuffle (X, Y)\n#     permutation = list(np.random.permutation(m))\n#     shuffled_X = X[permutation,:,:,:]\n#     shuffled_Y = Y[permutation,:]\n    permutation = np.random.permutation(m)\n    shuffled_X = X[permutation]\n    shuffled_Y = Y[permutation]\n\n    # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case.\n    i_start, i_end = 0, mini_batch_size\n    while i_start < m:\n        mini_batch_X = shuffled_X[i_start : i_end]\n        mini_batch_Y = shuffled_Y[i_start : i_end]\n\n        mini_batches.append((mini_batch_X, mini_batch_Y))\n#         mini_batches.append(mini_batch_X)\n\n        i_start += mini_batch_size\n        i_end += mini_batch_size\n    \n    return mini_batches","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"328157fe-344e-403b-abc4-248109b82c19","_uuid":"6af60d13e9d1181b6f4799fb9e3bbc9e4e577106","trusted":true,"collapsed":true},"cell_type":"code","source":"if not runInKeras:\n    learning_rate = 0.0003\n    num_epochs = 3\n    validation_split = 0.01\n    minibatch_size = 128\n\n    min_delta = 1e-3\n    patience = 20\n    min_cost = None\n\n\n    spliter = int(len(img_train) * (1-validation_split))\n    # X_train_, Y_train_, X_val, Y_val =  X_train[: spliter], Y_train[: spliter], X_train[spliter :], Y_train[spliter :]\n    img_train_, img_val = img_train[: spliter], img_train[spliter :]\n    X_val, Y_val = load_dataset('../input/train/', img_names = img_val)\n\n    # To keep track of the cost and accuracy\n    cost_list, acc_list, val_cost_list, val_acc_list = [], [], [], []\n\n\n    # to be able to rerun the model without overwriting tf variables\n    tf.reset_default_graph()\n\n    # Create Placeholders of the correct shape\n    X, Y = create_placeholders(IMG_SHAPE, 1)\n\n    # Forward propagation: Build the forward propagation in the tensorflow graph\n    Z, Y_pred = forward_propagation(X)\n\n    # Add cost and accuracy functions to tensorflow graph\n    cost = compute_cost(Z, Y, \"sigmoid\")\n    accuracy = compute_accuracy(Y_pred, Y, \"sigmoid\")\n\n    # Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.\n    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n\n    # Initialize all the variables globally\n    init = tf.global_variables_initializer()\n\n    # Start the session to compute the tensorflow graph\n    sess = tf.Session()\n\n    # Run the initialization\n    sess.run(init)\n\n    # Do the training loop\n    for epoch in range(1, num_epochs+1):\n        # Start-time used for printing time-usage below.\n        start_time = time.time()\n\n        train_cost, train_acc, test_cost, test_acc = 0., 0., 0., 0.\n\n    #     minibatches = random_mini_batches(X_train_, Y_train_, minibatch_size)\n        minibatches = random_mini_batches(img_train[: spliter], minibatch_size)\n        num_minibatches = len(minibatches) # number of minibatches of size minibatch_size in the train set\n\n        for minibatch in minibatches:\n            # Select a minibatch\n    #         minibatch_X, minibatch_Y = minibatch\n            minibatch_X, minibatch_Y = load_dataset('../input/train/', img_names = minibatch)\n\n            # IMPORTANT: The line that runs the graph on a minibatch.\n            # Run the session to execute the optimizer and the cost, the feedict should contain a minibatch for (X,Y).\n            _ , minibatch_cost, minibatch_acc = sess.run([optimizer, cost, accuracy], feed_dict={X: minibatch_X, Y: minibatch_Y})\n\n            train_cost += minibatch_cost / num_minibatches\n            train_acc += minibatch_acc / num_minibatches\n\n        val_cost, val_acc = sess.run([cost, accuracy], feed_dict={X: X_val, Y: Y_val})\n\n        cost_list.append(train_cost)\n        acc_list.append(train_acc)\n        val_cost_list.append(val_cost)\n        val_acc_list.append(val_acc)\n\n        # Ending time.\n        end_time = time.time()\n        # Difference between start and end-times.\n        time_dif = int(round(end_time - start_time))\n\n        # Print the cost every epoch\n        print(\"Epoch %i/%i - %is - cost: %f - acc %f - val_cost: %f - val_acc %f\" % (epoch, num_epochs, time_dif, train_cost, train_acc, val_cost, val_acc))\n\n        if min_cost is None or min_cost > val_cost:\n            min_cost, min_cost_epoch = val_cost, epoch\n\n        if epoch > min_cost_epoch + patience:\n            print(\"--- Early Stop ---\")\n            break\n\n    print(\"--- Completed ---\")\n\n    history = {\"loss\": cost_list, \"acc\": acc_list, \"val_loss\": val_cost_list, \"val_acc\": val_acc_list}","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0bbb37d0-7d15-4b24-b0e1-10cf0763fdfc","_uuid":"193593c48529b43a1b6a6cca91bc00138b49341d","trusted":true,"collapsed":true},"cell_type":"code","source":"if not runInKeras:\n    tf.trainable_variables()\n\n    # for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):\n    #     print(var)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c93cc2ab-efbb-48e4-8f33-667cc17d6580","collapsed":true,"_uuid":"701f138fc6e3f2fb683ce172b2819a0a00fa5e1c","trusted":true},"cell_type":"code","source":"if not runInKeras:\n    plot_train_history(history)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9ffdd07f-bd33-4a0d-b139-a20572cb7eb4","collapsed":true,"_uuid":"6c7c09c7bb893eeea69226575ab6a3a3830a45db","trusted":true},"cell_type":"code","source":"if not runInKeras:\n    test_accuracy = accuracy.eval({X: X_test, Y: Y_test}, session=sess)\n    print(\"Test Accuracy:\", test_accuracy)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"095c0784-47e7-423f-a46f-dfe6067ef0ad","collapsed":true,"_uuid":"8e4f87b5ddd449906b7feab113e46b1b465ff57d","trusted":true},"cell_type":"code","source":"if not runInKeras:\n    Y_test_pred = Y_pred.eval({X: X_test, Y: Y_test}, session=sess)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"ed6a85294158db5d9f7929890ee102b0795cd87b","trusted":false},"cell_type":"markdown","source":"**Analyze**"},{"metadata":{"_cell_guid":"8001069e-e778-484b-b3d5-c02b0e8df50d","_uuid":"a88c1175e6e4e6b3f60797e55ffe588bbee9b8c9","trusted":true,"collapsed":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, mean_squared_error, confusion_matrix, precision_score, recall_score, classification_report\n\ndef analyze(Y, Y_pred, classes, activation=\"softmax\"):\n    if activation == \"sigmoid\":\n        Y_cls = Y\n        Y_pred_cls = (Y_pred > 0.5).astype(float)\n    elif activation == \"softmax\":\n        Y_cls = np.argmax(Y, axis=1)\n        Y_pred_cls = np.argmax(Y_pred, axis=1)\n    \n    \n    accuracy = accuracy_score(Y_cls, Y_pred_cls)\n    print(\"Accuracy score: {}\\n\".format(accuracy))\n    \n    \n    rmse = np.sqrt(mean_squared_error(Y, Y_pred))\n    print(\"RMSE score: {}\\n\".format(rmse))\n\n    \n    # plot Confusion Matrix\n    print(\"Confusion Matrix:\")\n    cm = confusion_matrix(Y_cls, Y_pred_cls)\n    print(cm)\n    # Plot the confusion matrix as an image.\n    plt.matshow(cm)\n    # Make various adjustments to the plot.\n    num_classes = len(classes)\n    plt.colorbar()\n    tick_marks = np.arange(num_classes)\n    plt.xticks(tick_marks, range(num_classes))\n    plt.yticks(tick_marks, range(num_classes))\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n    \n    \n    # plot Classification Report\n    print(\"Classification Report:\")\n    print(classification_report(Y_cls, Y_pred_cls, target_names=classes))\n\n\n\nanalyze(Y_test, Y_test_pred, CLASSES, \"sigmoid\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8b69c458-7de3-4579-95bf-9de45f416516","scrolled":true,"_uuid":"c1c91419869c3e2003d678a7bf6fd8c6560d689d","trusted":true,"collapsed":true},"cell_type":"code","source":"def plot_mislabeled(X, Y, Y_pred, classes, activation=\"softmax\", num_images = 0):\n    \"\"\"\n    Plots images where predictions and truth were different.\n    \n    X -- original image data - shape(m, img_rows*img_cols)\n    Y -- true labels - eg. [2,3,4,3,1,1]\n    Y_pred -- predictions - eg. [2,3,4,3,1,2]\n    \"\"\"\n    \n    num_col = 5\n    \n    if activation == \"sigmoid\":\n        Y_cls = Y\n        Y_pred_cls = (Y_pred > 0.5).astype(float)\n    elif activation == \"softmax\":\n        Y_cls = np.argmax(Y, axis=1)\n        Y_pred_cls = np.argmax(Y_pred, axis=1)\n    \n    mislabeled_indices = np.where(Y_cls != Y_pred_cls)[0]\n    \n    if num_images < 1:\n        num_images = len(mislabeled_indices)\n    \n    fig, axes = plt.subplots(math.ceil(num_images/num_col), num_col, figsize=(25,20))\n\n    for i, index in enumerate(mislabeled_indices[:num_images]):\n#         plt.subplot(2, num_images, i + 1)\n#         plt.imshow(X[index, :].reshape(IMG_SHAPE[0], IMG_SHAPE[1], IMG_SHAPE[2]), interpolation='nearest')\n#         plt.axis('off')\n#         plt.title(\"Prediction: \" + classes[p[index]] + \" \\n Class: \" + classes[int(y[index])])\n        row, col = i//num_col, i%num_col\n        img = np.squeeze(X[index, :].reshape(IMG_SHAPE[0], IMG_SHAPE[1], IMG_SHAPE[2]))\n\n        axes[row, col].imshow(img, interpolation='nearest')\n        axes[row, col].axis('off')\n        axes[row, col].set_title(\"Id: {}\\nPrediction: {} - {}\\nClass: {}\".format(index, classes[int(Y_pred_cls[index])], np.amax(Y_pred[index]), classes[int(Y_cls[index])]))\n\n\n\nplot_mislabeled(X_test, Y_test, Y_test_pred, CLASSES, \"sigmoid\", 20)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"89eb534f-8ea7-4be2-86ac-1bf067cbaa55","_uuid":"0f523ff33527d014bfa1a104028f42ac6329148c","trusted":true,"scrolled":false,"collapsed":true},"cell_type":"code","source":"def plot_conv_weights(W):\n    _, _, channel_in, channel_out = W.shape\n    if channel_out > 10:\n        channel_out = 10\n    \n    fig, axes = plt.subplots(channel_out, channel_in, figsize=(20,20))\n\n    for row in range(channel_out):\n        for col in range(channel_in):\n            img = W[:, :, col, row]\n            axes[row, col].matshow(img, vmin=np.min(W), vmax=np.max(W), interpolation='nearest', cmap='seismic')\n\n\n\n            \nlayer_id = 2\n\nlayer_input = model.layers[layer_id]\nConv_W = layer_input.get_weights()[0]\n\n# Retrieve the values of the weight-variables from TensorFlow.\n# A feed-dict is not necessary because nothing is calculated.\n# Conv_W = sess.run(cache[\"Conv{}_W\".format(layer_id)])\n# with tf.variable_scope(\"Conv1\", reuse=True):\n#     Conv_W = tf.get_variable('Conv1_W')\n\nprint(Conv_W.shape)\nplot_conv_weights(Conv_W)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"542c7cf4-1b89-41cd-a166-5fc6ecd7aecb","_uuid":"22e62a3eb4b5a0d8a31b419e9a9d18bd679b337d","trusted":true,"collapsed":true},"cell_type":"code","source":"def plot_conv_layer(A):  \n    num_col = 5\n    \n    # Number of filters (channel_out) used in the conv. layer.\n    num_filters = A.shape[3]\n    if num_filters > 20:\n        num_filters = 20\n    \n    fig, axes = plt.subplots(math.ceil(num_filters/num_col), num_col, figsize=(20,20))\n\n    for i, ax in enumerate(axes.flat):\n        # Only plot the images for valid filters.\n        if i < num_filters:\n            img = A[0, :, :, i]\n            ax.matshow(img, interpolation='nearest', cmap='binary')\n#             ax.axis('off')\n            ax.set_xticks([])\n            ax.set_yticks([])\n\n\nimage_id = 101\nlayer_id = 3\n\nlayer_outputs = [layer.output for layer in model.layers]\nactivation_model = Model(inputs=model.input, outputs=layer_outputs)\nactivations = activation_model.predict(X_test[image_id : image_id+1])\nConv_A = activations[layer_id]\n\n# Calculate and retrieve the output values of the layer\n# when inputting that image.\n# Conv_A = sess.run(cache[\"Conv{}_A\".format(layer_id)], {X: X_test[image_id : image_id+1]})\n\nprint(Conv_A.shape)\nplot_image(X_test[image_id], Y_test[image_id], CLASSES)\nplot_conv_layer(Conv_A)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2bc21305419640c5c99b9fe861488f9740947cb","collapsed":true},"cell_type":"code","source":"def plot_conv_layers(image, model):\n    layer_names = [layer.name for layer in model.layers]\n    layer_outputs = [layer.output for layer in model.layers]\n    activation_model = Model(inputs=model.input, outputs=layer_outputs)\n    activations = activation_model.predict(np.expand_dims(image, axis=0))\n\n    images_per_row = 16\n    \n    for layer_name, layer_activation in zip(layer_names, activations):\n        if layer_name.startswith('Conv'):\n            _, height, width, num_filters = layer_activation.shape   # image height and width, and size of channel\n            n_rows = num_filters // images_per_row\n            display_grid = np.zeros((n_rows * height, images_per_row * width))\n\n            for row in range(n_rows):\n                for col in range(images_per_row):\n                    channel_image = layer_activation[0, :, :, row * images_per_row + col]\n                    channel_image -= channel_image.mean()\n                    channel_image /= channel_image.std()\n                    channel_image *= 64\n                    channel_image += 128\n                    channel_image = np.clip(channel_image, 0, 255).astype('uint8')\n\n                    display_grid[row * height : (row + 1) * height, col * width : (col + 1) * width] = channel_image\n\n            plt.figure(figsize=(images_per_row *2, n_rows *2))\n            plt.title(layer_name)\n            plt.grid(False)\n            plt.axis('off')\n            plt.imshow(display_grid, aspect='auto', interpolation='nearest', cmap='binary')\n            \n\n            \nimage_id = 101\nplot_image(X_test[image_id], Y_test[image_id], CLASSES)\nplot_conv_layers(X_test[image_id], model)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}