{"nbformat_minor": 1, "cells": [{"cell_type": "markdown", "source": ["**To begin with, get everything of data prepared before implementing the neural networks **"], "metadata": {"_cell_guid": "c785b61d-f9c6-46d3-9ead-4fc32de6f480", "_uuid": "c78944883567ee39ba6076cb78507a739dde99b4"}}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "6831331a-d41a-43fa-8f26-a363b9706314", "_uuid": "dea696f5688d6825efff89ae7464e3d6f81dfb67"}, "source": ["import os\n", "import time\n", "import sys\n", "import numpy as np\n", "import h5py\n", "import matplotlib.pyplot as plt\n", "import scipy\n", "import random\n", "import pandas as pd\n", "import warnings\n", "\n", "from PIL import Image\n", "from scipy import ndimage\n", "from itertools import chain\n", "from skimage.io import imread, imshow, imread_collection, concatenate_images\n", "from skimage.transform import resize\n", "from skimage.morphology import label\n", "from keras.models import Model, load_model\n", "from keras.layers import Input\n", "from keras.layers.core import Lambda\n", "from keras.layers.convolutional import Conv2D, Conv2DTranspose\n", "from keras.layers.pooling import MaxPooling2D\n", "from keras.layers.merge import concatenate\n", "from keras.callbacks import EarlyStopping, ModelCheckpoint\n", "from keras import backend as K\n", "from tqdm import tqdm\n", "\n", "%matplotlib inline\n", "plt.rcParams['figure.figsize'] = (5.0, 4.0) # set default size of plots\n", "plt.rcParams['image.interpolation'] = 'nearest'\n", "plt.rcParams['image.cmap'] = 'gray'\n", "\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "warnings.filterwarnings('ignore', category=UserWarning, module='skimage')"]}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "0eadbeb5-e4ac-446f-978f-6d032444148a", "_uuid": "f96ee809cb2bdfbaf9c61e716685663b9b567eaa"}, "source": ["# Set some parameters\n", "IMG_WIDTH = 128\n", "IMG_HEIGHT = 128\n", "IMG_CHANNELS = 3\n", "TRAIN_PATH = '../input/stage1_train/'\n", "TEST_PATH = '../input/stage1_test/'"]}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "11c8de6c-b82d-496f-a86a-96e5ec84df7b", "_uuid": "af54861b472af19f2d32d3a611f1d5214bf7686c"}, "source": ["# Get train and test IDs\n", "train_ids = next(os.walk(TRAIN_PATH))[1]\n", "test_ids = next(os.walk(TEST_PATH))[1]"]}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "d606bfc3-a60a-45a1-a35e-a8f6283a7ede", "_uuid": "3b636cb841d4bec9fd939c2a37a13fc37d0f90d3"}, "source": ["# Get and resize train images and masks\n", "X_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\n", "Y_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\n", "print('Getting and resizing train images and masks ... ')\n", "sys.stdout.flush()\n", "for n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):\n", "    path = TRAIN_PATH + id_\n", "    img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n", "    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n", "    X_train[n] = img\n", "    mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)\n", "    for mask_file in next(os.walk(path + '/masks/'))[2]:\n", "        mask_ = imread(path + '/masks/' + mask_file)\n", "        mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode='constant', \n", "                                      preserve_range=True), axis=-1)\n", "        mask = np.maximum(mask, mask_)\n", "    Y_train[n] = mask\n", "\n", "# Get and resize test images\n", "X_test = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)\n", "sizes_test = []\n", "print('Getting and resizing test images ... ')\n", "sys.stdout.flush()\n", "for n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):\n", "    path = TEST_PATH + id_\n", "    img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]\n", "    sizes_test.append([img.shape[0], img.shape[1]])\n", "    img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)\n", "    X_test[n] = img\n", "\n", "print('Done!')"]}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "420158b8-56db-46f9-9969-8981466c23c8", "_uuid": "500d0c3cf15b51955d4c2e4826803945446cef08"}, "source": ["# Check if training data looks all right\n", "ix = random.randint(0, len(train_ids))\n", "imshow(X_train[ix])\n", "plt.show()\n", "imshow(np.squeeze(Y_train[ix]))\n", "plt.show()"]}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "d216727d-8676-4ad4-933b-8ba9a10f83f8", "_uuid": "5a3bb6f7c24ba22abd4638497d5298246bdf193a"}, "source": ["# check the size of dataset \n", "m_train = X_train.shape[0]\n", "num_px = X_train.shape[1]\n", "m_test = X_test.shape[0]\n", "\n", "print (\"Number of training examples: \" + str(m_train))\n", "print (\"Number of testing examples: \" + str(m_test))\n", "print (\"Each image is of size: (\" + str(num_px) + \", \" + str(num_px) + \", 3)\")\n", "print (\"train_x_orig shape: \" + str(X_train.shape))\n", "print (\"train_y shape: \" + str(Y_train.shape))\n", "print (\"test_x_orig shape: \" + str(X_test.shape))"]}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "37373596-3946-4bfc-b36d-1d233a066841", "_uuid": "cc45e65a1efd07cddc30988815a63530be9f694f"}, "source": ["# Reshape the training and test examples \n", "train_x_flatten = X_train.reshape(X_train.shape[0], -1).T # The \"-1\" makes reshape flatten the remaining dimensions \n", "test_x_flatten = X_test.reshape(X_test.shape[0], -1).T\n", "    \n", "# Standardize data to have feature values between 0 and 1.\n", "train_x = train_x_flatten/255.\n", "test_x = test_x_flatten/255.\n", "\n", "print (\"train_x's shape: \" + str(train_x.shape))\n", "print (\"test_x's shape: \" + str(test_x.shape))"]}, {"cell_type": "markdown", "source": ["**Define Basic Functions**"], "metadata": {}}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true}, "source": ["# FUNCTION: sigmoid\n", "def sigmoid(x):\n", "    \n", "    s = 1/(1+np.exp(-x))\n", "    \n", "    return s\n", "\n", "# FUNCTION: relu\n", "def relu(x):\n", "    s = max(0,x)\n", "    return s\n", "\n", "# FUNCTION: initialize_parameters\n", "def initialize_parameters(n_x, n_h, n_y):\n", "    W1 = np.random.randn(n_h,n_x)*0.01\n", "    b1 = np.zeros((n_h, 1))\n", "    W2 = np.random.randn(n_y,n_h)*0.01\n", "    b2 = np.zeros((n_y,1))\n", "    assert(W1.shape == (n_h, n_x))\n", "    assert(b1.shape == (n_h, 1))\n", "    assert(W2.shape == (n_y, n_h))\n", "    assert(b2.shape == (n_y, 1))\n", "    parameters = {\"W1\": W1,\n", "                  \"b1\": b1,\n", "                  \"W2\": W2,\n", "                  \"b2\": b2}\n", "    return parameters    \n", "\n", "# FUNCTION: linear_forward\n", "def linear_forward(A, W, b):\n", "    Z = np.dot(W,A) + b\n", "    assert(Z.shape == (W.shape[0], A.shape[1]))\n", "    cache = (A, W, b)\n", "    return Z, cache\n", "\n", "# FUNCTION: compute_cost\n", "def compute_cost(AL, Y):\n", "    m = Y.shape[1]\n", "    cost = -1/m* np.sum(np.multiply(np.log(AL),Y) + np.multiply(np.log(1 - AL),1 - Y))\n", "    cost = np.squeeze(cost)    \n", "    assert(cost.shape == ())\n", "    return cost\n", "\n", "# FUNCTION: linear_backward\n", "\n", "def linear_backward(dZ, cache):\n", "    A_prev, W, b = cache\n", "    m = A_prev.shape[1]\n", "    dW = 1/m * np.dot(dZ,A_prev.T)\n", "    db = np.matrix(1/m * np.sum(dZ))\n", "    dA_prev = np.dot(W.T,dZ)\n", "    assert (dA_prev.shape == A_prev.shape)\n", "    assert (dW.shape == W.shape)\n", "    assert (db.shape == b.shape)\n", "    return dA_prev, dW, db\n", "\n", "def update_parameters(parameters, grads, learning_rate):\n", "    L = len(parameters) // 2 \n", "    W1 = parameters['W1']\n", "    b1 = parameters['b1']\n", "    W2 = parameters['W2']\n", "    b2 = parameters['b2']\n", "    dW1 = grads['dW1']\n", "    db1 = grads['db1']\n", "    dW2 = grads['dW2']\n", "    db2 = grads['db2']\n", "    W1 = W1 - learning_rate * dW1\n", "    b1 = b1 - learning_rate * db1\n", "    W2 = W2 - learning_rate * dW2\n", "    b2 = b2 - learning_rate * db2\n", "    parameters = {\"W1\": W1,\n", "                  \"b1\": b1,\n", "                  \"W2\": W2,\n", "                  \"b2\": b2}\n", "    return parameters"]}, {"cell_type": "markdown", "source": ["**First, Try a Two-layer neural network**"], "metadata": {"_cell_guid": "cb7bc12f-7d79-4899-8ba2-accf52e32e6c", "_uuid": "0d175c46fc27adc51fa47f5b676d1fcbb088e309"}}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true}, "source": ["### CONSTANTS DEFINING THE MODEL ####\n", "n_x = 49152     # num_px * num_px * 3\n", "n_h = 7\n", "n_y = 1\n", "layers_dims = (n_x, n_h, n_y)"]}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true}, "source": ["# FUNCTION: two_layer_model\n", "\n", "def two_layer_model(X, Y, layers_dims, learning_rate = 0.0075, num_iterations = 3000, print_cost=False):\n", "    \n", "    grads = {}\n", "    costs = []                              \n", "    m = X.shape[1]                           \n", "    (n_x, n_h, n_y) = layers_dims\n", "    \n", "    parameters = initialize_parameters(n_x, n_h, n_y)\n", "    \n", "    W1 = parameters[\"W1\"]\n", "    b1 = parameters[\"b1\"]\n", "    W2 = parameters[\"W2\"]\n", "    b2 = parameters[\"b2\"]\n", "    \n", "    for i in range(0, num_iterations):\n", "\n", "        Z1, linear_cache1 = linear_forward(X, W1, b1)\n", "        A1,activation_cache1 = relu(Z1)\n", "        cache1 = (linear_cache1, activation_cache1)\n", "        Z2, linear_cache2 =  linear_forward(A1, W2, b2)\n", "        A2, activation_cache2 = sigmoid(Z2)\n", "        cache2 = (linear_cache2, activation_cache2)\n", "        \n", "        cost = compute_cost(A2, Y)\n", "    \n", "        dA2 = - (np.divide(Y, A2) - np.divide(1 - Y, 1 - A2))\n", "        \n", "        #......\n", "        linear_cache2, activation_cache2 = cache2\n", "        dZ1 = sigmoid_backward(dA2, activation_cache2)\n", "        dA1, dW2, db2 = linear_backward(dZ1, linear_cache2)\n", "        \n", "        linear_cache1, activation_cache1 = cache1\n", "        dZ2 = relu_backward(dA1, activation_cache1)\n", "        dA0, dW1, db1 = linear_backward(dZ2, linear_cache1)\n", "        \n", "        \n", "        grads['dW1'] = dW1\n", "        grads['db1'] = db1\n", "        grads['dW2'] = dW2\n", "        grads['db2'] = db2\n", "        \n", "\n", "        parameters = update_parameters(parameters, grads, learning_rate)\n", "\n", "        W1 = parameters[\"W1\"]\n", "        b1 = parameters[\"b1\"]\n", "        W2 = parameters[\"W2\"]\n", "        b2 = parameters[\"b2\"]\n", "        \n", "        if print_cost and i % 100 == 0:\n", "            print(\"Cost after iteration {}: {}\".format(i, np.squeeze(cost)))\n", "        if print_cost and i % 100 == 0:\n", "            costs.append(cost)\n", "       \n", "\n", "    plt.plot(np.squeeze(costs))\n", "    plt.ylabel('cost')\n", "    plt.xlabel('iterations (per tens)')\n", "    plt.title(\"Learning rate =\" + str(learning_rate))\n", "    plt.show()\n", "    \n", "    return parameters"]}, {"cell_type": "markdown", "source": ["**Then, try a DNN by tensorFlow**"], "metadata": {}}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true}, "source": ["# Load dataset and start training\n", "\n", "train_subset = 10000  \n", "  \n", "graph = tf.Graph()  \n", "with graph.as_default():  \n", "    # Input data.                    \n", "    # Load the training, validation and test data into constants that are  \n", "    # attached to the graph.  \n", "    tf_train_dataset = tf.constant(train_x[:train_subset, :])  \n", "    tf_train_labels = tf.constant(train_labels[:train_subset])  \n", "      \n", "    tf_valid_dataset = tf.constant(valid_dataset)  \n", "    tf_test_dataset = tf.constant(test_dataset)  \n", "    \n", "    # Variables.\u5b9a\u4e49\u53d8\u91cf \u8981\u8bad\u7ec3\u5f97\u5230\u7684\u53c2\u6570weight, bias  ----------------------------------------2  \n", "    # These are the parameters that we are going to be training. The weight  \n", "    # matrix will be initialized using random values following a (truncated)  \n", "    # normal distribution. The biases get initialized to zero.  \n", "    weights = tf.Variable(tf.truncated_normal([image_size * image_size, num_labels])) # changing when training   \n", "    biases = tf.Variable(tf.zeros([num_labels])) # changing when training   \n", "      \n", "    #   tf.truncated_normal  \n", "    #   tf.truncated_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None, name=None)  \n", "    #   Outputs random values from a truncated normal distribution.  \n", "    #  The generated values follow a normal distribution with specified mean and  \n", "    #  standard deviation, except that values whose magnitude is more than 2 standard  \n", "    #  deviations from the mean are dropped and re-picked.  \n", "      \n", "    # tf.zeros  \n", "    #  tf.zeros([10])      <tf.Tensor 'zeros:0' shape=(10,) dtype=float32>  \n", "  \n", "  \n", "    \n", "    # Training computation. \u8bad\u7ec3\u6570\u636e                                ----------------------------------------3  \n", "    # We multiply the inputs with the weight matrix, and add biases. We compute  \n", "    # the softmax and cross-entropy (it's one operation in TensorFlow, because  \n", "    # it's very common, and it can be optimized). We take the average of this  \n", "    # cross-entropy across all training examples: that's our loss.  \n", "    logits = tf.matmul(tf_train_dataset, weights) + biases             # tf.matmul          matrix multiply       \n", "      \n", "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))  # compute average cross entropy loss  \n", "    #  softmax_cross_entropy_with_logits  \n", "      \n", "    # The activation ops provide different types of nonlinearities for use in neural  \n", "    # networks.  These include smooth nonlinearities (`sigmoid`, `tanh`, `elu`,  \n", "    #   `softplus`, and `softsign`), continuous but not everywhere differentiable  \n", "    # functions (`relu`, `relu6`, and `relu_x`), and random regularization (`dropout`).  \n", "      \n", "      \n", "    #  tf.reduce_mean  \n", "    #    tf.reduce_mean(input_tensor, reduction_indices=None, keep_dims=False, name=None)  \n", "    #   Computes the mean of elements across dimensions of a tensor.  \n", "    \n", "    # Optimizer.                                                                    -----------------------------------------4  \n", "    # We are going to find the minimum of this loss using gradient descent.  \n", "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)     # 0.5 means learning rate  \n", "    #  tf.train.GradientDescentOptimizer(  \n", "    #  tf.train.GradientDescentOptimizer(self, learning_rate, use_locking=False, name='GradientDescent')  \n", "      \n", "      \n", "      \n", "    \n", "    # Predictions for the training, validation, and test data.---------------------------------------5  \n", "    # These are not part of training, but merely here so that we can report  \n", "    # accuracy figures as we train.  \n", "      \n", "    train_prediction = tf.nn.softmax(logits) # weights  and bias have been changed  \n", "    valid_prediction = tf.nn.softmax(tf.matmul(tf_valid_dataset, weights) + biases)  \n", "    test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)  \n", "      \n", "    # tf.nn.softmax  \n", "    #  Returns: A `Tensor`. Has the same type as `logits`. Same shape as `logits`.(num, 784) *(784,10)  + = (num, 10)  \n", "\n", "    "]}, {"cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true}, "source": []}, {"cell_type": "markdown", "source": [], "metadata": {"_cell_guid": "b2c1c223-bbe7-444a-addc-c83d9aba029d", "_uuid": "1ff8e4978b1aed6d93058530c64113843dd67f22"}}], "nbformat": 4, "metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"name": "python", "mimetype": "text/x-python", "pygments_lexer": "ipython3", "file_extension": ".py", "codemirror_mode": {"name": "ipython", "version": 3}, "version": "3.6.4", "nbconvert_exporter": "python"}}}