{"cells": [{"metadata": {}, "source": ["### Table of Contents\n", "\n", "1. Overview of the Problem set\n", "\n", "2. Load and prep the data\n", "\n", "3. Building the parts of our algorithm.\n", "\n", "4. Merge all functions into a model\n", "\n", "5. Visualizing and Analysis\n", "\n", "6. Lab"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["import os, math\n", "import random\n", "import sys\n", "import datetime\n", "## pip3 install opencv-python\n", "import cv2\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {"collapsed": true}, "source": ["## constants\n", "TRAIN_DIR = \"./input/train/\"\n", "TEST_DIR = \"./input/test/\"\n", "TRAIN_SIZE = 25000\n", "TEST_SIZE = 12500\n", "DEV_RATIO = 0.1\n", "IMAGE_HEIGHT = IMAGE_WIDTH = 128\n", "\n", "LEARNING_RATE = 0.0001\n", "MINIBATCH_SIZE = 32\n", "INPUT_SIZE = IMAGE_HEIGHT * IMAGE_WIDTH * 3\n", "OUTPUT_SIZE = 2"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["### 1. Overview of the Problem set\n", "A fun project to differentiate dogs from cats. Dataset is from Kaggle: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition.\n", "\n", "The ./input/train/ dir contains 12500 cat images and 12500 dog images. Each filename contains \"cat\" or \"dog\" as label.\n", "\n", "The ./input/test/ dir contains 12500 images to classify\n"], "cell_type": "markdown"}, {"metadata": {}, "source": ["### 2. Load and prep the data"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["## data utility functions\n", "def split_data(two_dims_datas, split_ratio=DEV_RATIO):\n", " left_count = int(two_dims_datas.shape[1] * split_ratio)\n", " left_datas = two_dims_datas[:, :left_count]\n", " right_datas = two_dims_datas[:, left_count:]\n", " print(\"input datas shape: {}, left datas shape:{}, \\\n", " right datas shape: {}\".format(two_dims_datas.shape, left_datas.shape, right_datas.shape))\n", " return left_datas, right_datas"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["S = np.array([1, 2, 2])\n", "S.astype(np.float)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {"collapsed": true}, "source": ["def load_images(dirname=\"./input/train/\", file_count=1000, shuffle=True,\n", " image_width=128, image_height=128, image_channels=3):\n", " \"\"\"\n", " Arguments:\n", " dirname -- dirname which contains image files.Str.Default TRAIN_DIR\n", " file_count -- number of files want to load.\n", " shuffle -- if True, returns images and labels have shuffled.\n", " image_width -- Image widht.Int .Default 128.\n", " image_height -- Image height.Int .Default 128.\n", " image_channels -- number of Image channel.Int .Default 3.\n", " \n", " Returns:\n", " images -- numpy array containing images data.\n", " (n_x, m) = (image_width * image_height * image_channels, m)\n", " labels -- numpy array containing image labels.[if is_dog: 1].(1, m)\n", " \"\"\"\n", " all_filenames = os.listdir(dirname)\n", " random.shuffle(all_filenames)\n", " filenames = all_filenames[:file_count]\n", " \n", " ## images\n", " images = np.zeros((file_count, IMAGE_HEIGHT*IMAGE_WIDTH*3))\n", " for i in range(file_count):\n", " imgnd_origin = cv2.imread(dirname+filenames[i])\n", " imgnd_resized = cv2.resize(imgnd_origin, (IMAGE_HEIGHT, IMAGE_WIDTH), interpolation=cv2.INTER_CUBIC)\n", " imgnd_flatten = imgnd_resized.reshape(1,-1)\n", " images[i] = imgnd_flatten\n", " \n", " ## labels from filenames\n", " labels_list = [\"dog\" in filename for filename in filenames]\n", " labels = np.array(labels_list, dtype='int8').reshape(file_count, 1) ## 1 if dog\n", " \n", " ## shuffle\n", " if shuffle:\n", " permutation = list(np.random.permutation(labels.shape[0]))\n", " labels = labels[permutation, :]\n", " images = images[permutation, :]\n", "\n", " ## normalization\n", " images = images/255.0\n", " images.astype(np.float)\n", " \n", " return images.T, labels.T\n", "\n", "def random_mini_batches(X, Y, mini_batch_size = 64, shuffle=True):\n", " \"\"\"\n", " Creates a list of random minibatches from (X, Y)\n", " \n", " Arguments:\n", " X -- input data, of shape (input size, number of examples)\n", " Y -- true \"label\" vector (1 for blue dot / 0 for red dot), of shape (1, number of examples)\n", " mini_batch_size -- size of the mini-batches, integer\n", " \n", " Returns:\n", " mini_batches -- list of synchronous (mini_batch_X, mini_batch_Y)\n", " \"\"\"\n", " \n", " m = X.shape[1] # number of training examples\n", " mini_batches = []\n", " \n", " # Step 1: Shuffle (X, Y)\n", " if shuffle:\n", " permutation = list(np.random.permutation(m))\n", " X = X[:, permutation]\n", " Y = Y[:, permutation].reshape((1,m))\n", "\n", " # Step 2: Partition (X, Y). Minus the end case.\n", " num_complete_minibatches = math.floor(m/mini_batch_size) # number of mini batches of size mini_batch_size in your partitionning\n", " for k in range(0, num_complete_minibatches):\n", " mini_batch_X = X[:, mini_batch_size*k:mini_batch_size*(k+1)]\n", " mini_batch_Y = Y[:, mini_batch_size*k:mini_batch_size*(k+1)]\n", " \n", " mini_batch = (mini_batch_X, mini_batch_Y)\n", " mini_batches.append(mini_batch)\n", " \n", " # Handling the end case (last mini-batch < mini_batch_size)\n", " if m % mini_batch_size != 0:\n", " mini_batch_X = X[:, -(m-mini_batch_size* math.floor(m/mini_batch_size)):]\n", " mini_batch_Y = Y[:, -(m-mini_batch_size* math.floor(m/mini_batch_size)):]\n", " \n", " mini_batch = (mini_batch_X, mini_batch_Y)\n", " mini_batches.append(mini_batch)\n", " \n", " return mini_batches"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["images, labels = load_images(file_count=100)\n", "dev_images, train_images = split_data(images)\n", "dev_labels, train_labels = split_data(labels)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["### 3. Building the parts of our algorithm.\n", " The model can be summarized as: [LINEAR -> RELU] \u00d7\u00d7 (L-1) -> LINEAR -> SIGMOID\n", " \n", "\n", "\n", "As usual you will follow the Deep Learning methodology to build the model:\n", " 1. Initialize parameters / Define hyperparameters\n", " 2. Loop for num_iterations:\n", " a. Forward propagation\n", " b. Compute cost function\n", " c. Backward propagation\n", " d. Update parameters (using parameters, and grads from backprop) \n", " 4. Use trained parameters to predict labels\n", "\n", "Let's now implement those two models!\n"], "cell_type": "markdown"}, {"metadata": {}, "source": ["** 3.1 Initialize the parameters for an LL-layer neural network **\n", "\n", "**Instructions**:\n", "- The model's structure is *[LINEAR -> RELU] $ \\times$ (L-1) -> LINEAR -> SIGMOID*. I.e., it has $L-1$ layers using a ReLU activation function followed by an output layer with a sigmoid activation function.\n", "- Use random initialization for the weight matrices. Use `np.random.rand(shape) * 0.01`.\n", "- Use zeros initialization for the biases. Use `np.zeros(shape)`.\n", "- We will store $n^{[l]}$, the number of units in different layers, in a variable `layer_dims`. For example, the `layer_dims` for the \"Planar Data classification model\" from last week would have been [2,4,1]: There were two inputs, one hidden layer with 4 hidden units, and an output layer with 1 output unit. Thus means `W1`'s shape was (4,2), `b1` was (4,1), `W2` was (1,4) and `b2` was (1,1). Now you will generalize this to $L$ layers! \n", "- Here is the implementation for $L=1$ (one layer neural network). It should inspire you to implement the general case (L-layer neural network).\n", "```python\n", " if L == 1:\n", " parameters[\"W\" + str(L)] = np.random.randn(layer_dims[1], layer_dims[0]) * 0.01\n", " parameters[\"b\" + str(L)] = np.zeros((layer_dims[1], 1))\n", "```"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["def initialize_parameters_deep(layer_dims):\n", " \"\"\"\n", " Arguments:\n", " layer_dims -- python array (list) containing the dimensions of each layer in our network\n", " \n", " Returns:\n", " parameters -- python dictionary containing your parameters \"W1\", \"b1\", ..., \"WL\", \"bL\":\n", " Wl -- weight matrix of shape (layer_dims[l], layer_dims[l-1])\n", " bl -- bias vector of shape (layer_dims[l], 1)\n", " \"\"\"\n", " \n", " parameters = {}\n", " L = len(layer_dims) # number of layers in the network\n", "\n", " for l in range(1, L):\n", " parameters['W' + str(l)] = np.random.randn(layer_dims[l], layer_dims[l-1]) / np.sqrt(layer_dims[l-1]) #*0.01\n", " parameters['b' + str(l)] = np.zeros((layer_dims[l], 1))\n", " \n", " assert(parameters['W' + str(l)].shape == (layer_dims[l], layer_dims[l-1]))\n", " assert(parameters['b' + str(l)].shape == (layer_dims[l], 1))\n", "\n", " \n", " return parameters"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["** 3.2 Implement the forward propagation module **\n", "\n", "- 3.2.1 Linear Forward\n", "\n", " **Reminder**:\n", " The mathematical representation of this unit is $Z^{[l]} = W^{[l]}A^{[l-1]} +b^{[l]}$."], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["## Linear Forward\n", "def linear_forward(A_prev, W, b):\n", " \"\"\"\n", " Implement the linear part of a layer's forward propagation.\n", "\n", " Arguments:\n", " A_prev -- activations from previous layer (or input data): (size of previous layer, number of examples)\n", " W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)\n", " b -- bias vector, numpy array of shape (size of the current layer, 1)\n", "\n", " Returns:\n", " Z -- the input of the activation function, also called pre-activation parameter \n", " cache -- a python dictionary containing \"A\", \"W\" and \"b\" ; stored for computing the backward pass efficiently\n", " \"\"\"\n", " \n", " Z = np.add(np.matmul(W, A_prev), b)\n", " \n", " assert(Z.shape == (W.shape[0], A_prev.shape[1]))\n", " cache = (A_prev, W, b)\n", " \n", " return Z, cache"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["- 3.2.2 Linear-Activation Forward\n", "\n", " **Reminder**:\n", " Forward propagation of the *LINEAR->ACTIVATION* layer. Mathematical relation is: $A^{[l]} = g(Z^{[l]}) = g(W^{[l]}A^{[l-1]} +b^{[l]})$ where the activation \"g\" can be sigmoid() or relu()."], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["def sigmoid(Z):\n", " \"\"\"\n", " Implements the sigmoid activation in numpy\n", " \n", " Arguments:\n", " Z -- numpy array of any shape\n", " \n", " Returns:\n", " A -- output of sigmoid(z), same shape as Z\n", " cache -- returns Z as well, useful during backpropagation\n", " \"\"\"\n", " \n", " A = 1/(1+np.exp(-Z))\n", " cache = Z\n", " \n", " return A, cache\n", "\n", "def relu(Z):\n", " \"\"\"\n", " Implement the RELU function.\n", "\n", " Arguments:\n", " Z -- Output of the linear layer, of any shape\n", "\n", " Returns:\n", " A -- Post-activation parameter, of the same shape as Z\n", " cache -- a python dictionary containing \"A\" ; stored for computing the backward pass efficiently\n", " \"\"\"\n", " \n", " A = np.maximum(0,Z)\n", " \n", " assert(A.shape == Z.shape)\n", " \n", " cache = Z \n", " return A, cache\n", "\n", "## Linear-Activation Forward\n", "def linear_activation_forward(A_prev, W, b, activation):\n", " \"\"\"\n", " Implement the forward propagation for the LINEAR->ACTIVATION layer\n", "\n", " Arguments:\n", " A_prev -- activations from previous layer (or input data): (size of previous layer, number of examples)\n", " W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)\n", " b -- bias vector, numpy array of shape (size of the current layer, 1)\n", " activation -- the activation to be used in this layer, stored as a text string: \"sigmoid\" or \"relu\"\n", "\n", " Returns:\n", " A -- the output of the activation function, also called the post-activation value \n", " cache -- a python dictionary containing \"linear_cache\" and \"activation_cache\";\n", " stored for computing the backward pass efficiently\n", " \"\"\"\n", " \n", " if activation == \"sigmoid\":\n", " # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\".\n", " Z, linear_cache = linear_forward(A_prev, W, b)\n", " #print(57, \"Z, linear_cache = linear_forward(A_prev, W, b)\", Z)\n", " A, activation_cache = sigmoid(Z)\n", " #print(58, \"A, activation_cache = sigmoid(Z)\", A)\n", " \n", " elif activation == \"relu\":\n", " # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\".\n", " Z, linear_cache = linear_forward(A_prev, W, b)\n", " A, activation_cache = relu(Z)\n", " \n", " assert (A.shape == (W.shape[0], A_prev.shape[1]))\n", " cache = (linear_cache, activation_cache)\n", "\n", " return A, cache"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["- 3.2.3 L_model_forward\n", "\n", " For even more convenience when implementing the $L$-layer Neural Net, you will need a function that replicates the previous one (`linear_activation_forward` with RELU) $L-1$ times, then follows that with one `linear_activation_forward` with SIGMOID.\n", " \n", " \n", "
**Figure 2** : *[LINEAR -> RELU] $\\times$ (L-1) -> LINEAR -> SIGMOID* model

"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["## L-Layer Model\n", "def L_model_forward(X, parameters):\n", " \"\"\"\n", " Implement forward propagation for the [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID computation\n", " \n", " Arguments:\n", " X -- data, numpy array of shape (input size, number of examples)\n", " parameters -- output of initialize_parameters_deep()\n", " \n", " Returns:\n", " AL -- last post-activation value\n", " caches -- list of caches containing:\n", " every cache of linear_activation_forward(A_prev, W, b, 'relu') (there are L-1 of them, indexed from 0 to L-2)\n", " the cache of linear_activation_forward(A_prev, W, b, 'sigmoid') (there is one, indexed L-1)\n", " \"\"\"\n", "\n", " caches = []\n", " A = X\n", " L = len(parameters) // 2 # number of layers in the neural network\n", " \n", " # Implement [LINEAR -> RELU]*(L-1). Add \"cache\" to the \"caches\" list.\n", " #print(\"isnan in X\",np.argwhere(np.isnan(X)))\n", " for l in range(1, L):\n", " A_prev = A \n", " A, cache = linear_activation_forward(\n", " A_prev, parameters['W' + str(l)], parameters['b' + str(l)], 'relu'\n", " )\n", " caches.append(cache)\n", " #print(28, \"linear_activation_forward\", l, A)\n", " \n", " # Implement LINEAR -> SIGMOID. Add \"cache\" to the \"caches\" list.\n", " AL, cache = linear_activation_forward(\n", " A, parameters['W' + str(L)], parameters['b' + str(L)], 'sigmoid'\n", " )\n", " caches.append(cache)\n", " #print(\"isnan in AL\",np.argwhere(np.isnan(AL)))\n", " #print(\"37 AL equal 1\", np.argwhere(AL==1))\n", " assert(AL.shape == (1,X.shape[1]))\n", " AL = np.minimum(AL, 0.99999)\n", " \n", " return AL, caches"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["** 3.3 Compute the loss **\n", "\n", "Compute the cross-entropy cost $J$, using the following formula: $$-\\frac{1}{m} \\sum\\limits_{i = 1}^{m} (y^{(i)}\\log\\left(a^{[L] (i)}\\right) + (1-y^{(i)})\\log\\left(1- a^{[L](i)}\\right))\u00a0\\tag{7}$$"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["def compute_cost(AL, Y):\n", " \"\"\"\n", " Implement the cost function defined by equation (7).\n", "\n", " Arguments:\n", " AL -- probability vector corresponding to your label predictions, shape (1, number of examples)\n", " Y -- true \"label\" vector (for example: containing 0 if non-cat, 1 if cat), shape (1, number of examples)\n", "\n", " Returns:\n", " cost -- cross-entropy cost\n", " \"\"\"\n", " \n", " m = Y.shape[1]\n", "\n", " # Compute loss from aL and y.\n", " cost = (1./m) * (-np.dot(Y,np.log(AL).T) - np.dot(1-Y, np.log(1-AL).T))\n", " \n", " cost = np.squeeze(cost) # To make sure your cost's shape is what we expect (e.g. this turns [[17]] into 17).\n", " assert(cost.shape == ())\n", " \n", " return cost"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["** 3.4 Implement the backward propagation module **\n", "\n", "**Reminder**: \n", "\n", "
**Figure 3** : Forward and Backward propagation for *LINEAR->RELU->LINEAR->SIGMOID*
*The purple blocks represent the forward propagation, and the red blocks represent the backward propagation.*
"], "cell_type": "markdown"}, {"metadata": {}, "source": ["- 3.4.1 Linear backward\n", "\n", " For layer $l$, the linear part is: $Z^{[l]} = W^{[l]} A^{[l-1]} + b^{[l]}$ (followed by an activation).\n", "\n", " Suppose you have already calculated the derivative $dZ^{[l]} = \\frac{\\partial \\mathcal{L} }{\\partial Z^{[l]}}$. You want to get $(dW^{[l]}, db^{[l]} dA^{[l-1]})$.\n", "\n", " \n", "
**Figure 4**
\n", "\n", " The three outputs $(dW^{[l]}, db^{[l]}, dA^{[l]})$ are computed using the input $dZ^{[l]}$.Here are the formulas you need:\n", " $$ dW^{[l]} = \\frac{\\partial \\mathcal{L} }{\\partial W^{[l]}} = \\frac{1}{m} dZ^{[l]} A^{[l-1] T} \\tag{8}$$\n", " $$ db^{[l]} = \\frac{\\partial \\mathcal{L} }{\\partial b^{[l]}} = \\frac{1}{m} \\sum_{i = 1}^{m} dZ^{[l](i)}\\tag{9}$$\n", " $$ dA^{[l-1]} = \\frac{\\partial \\mathcal{L} }{\\partial A^{[l-1]}} = W^{[l] T} dZ^{[l]} \\tag{10}$$"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["## Linear backward\n", "def linear_backward(dZ, cache):\n", " \"\"\"\n", " Implement the linear portion of backward propagation for a single layer (layer l)\n", "\n", " Arguments:\n", " dZ -- Gradient of the cost with respect to the linear output (of current layer l)\n", " cache -- tuple of values (A_prev, W, b) coming from the forward propagation in the current layer\n", "\n", " Returns:\n", " dA_prev -- Gradient of the cost with respect to the activation (of the previous layer l-1), same shape as A_prev\n", " dW -- Gradient of the cost with respect to W (current layer l), same shape as W\n", " db -- Gradient of the cost with respect to b (current layer l), same shape as b\n", " \"\"\"\n", " A_prev, W, b = cache\n", " m = A_prev.shape[1]\n", "\n", " ##### A_prev.T.shape = (m, n_(L-1)), dZ.shape = (1, n_L)\n", " dW = (1.0/m) * np.matmul(dZ, A_prev.T)\n", " db = (1.0/m) * np.sum(dZ, axis=-1, keepdims=True)\n", " dA_prev = np.matmul(np.transpose(W), dZ)\n", " \n", " assert (dA_prev.shape == A_prev.shape)\n", " assert (dW.shape == W.shape)\n", " assert (db.shape == b.shape)\n", " \n", " return dA_prev, dW, db"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["- 3.4.2 Linear-Activation backward\n", "\n", "If $g(.)$ is the activation function, \n", "`sigmoid_backward` and `relu_backward` compute $$dZ^{[l]} = dA^{[l]} * g'(Z^{[l]}) \\tag{11}$$. "], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["def sigmoid_backward(dA, cache):\n", " \"\"\"\n", " Implement the backward propagation for a single SIGMOID unit.\n", "\n", " Arguments:\n", " dA -- post-activation gradient, of any shape\n", " cache -- 'Z' where we store for computing backward propagation efficiently\n", "\n", " Returns:\n", " dZ -- Gradient of the cost with respect to Z\n", " \"\"\"\n", " \n", " Z = cache\n", " \n", " s = 1/(1+np.exp(-Z))\n", " dZ = dA * s * (1-s)\n", " \n", " assert (dZ.shape == Z.shape)\n", " \n", " return dZ\n", "\n", "def relu_backward(dA, cache):\n", " \"\"\"\n", " Implement the backward propagation for a single RELU unit.\n", "\n", " Arguments:\n", " dA -- post-activation gradient, of any shape\n", " cache -- 'Z' where we store for computing backward propagation efficiently\n", "\n", " Returns:\n", " dZ -- Gradient of the cost with respect to Z\n", " \"\"\"\n", " \n", " Z = cache\n", " dZ = np.array(dA, copy=True) # just converting dz to a correct object.\n", " \n", " # When z <= 0, you should set dz to 0 as well. \n", " dZ[Z <= 0] = 0\n", " \n", " assert (dZ.shape == Z.shape)\n", " \n", " return dZ\n", "\n", "## Linear-Activation backward\n", "def linear_activation_backward(dA, cache, activation):\n", " \"\"\"\n", " Implement the backward propagation for the LINEAR->ACTIVATION layer.\n", " \n", " Arguments:\n", " dA -- post-activation gradient for current layer l \n", " cache -- tuple of values (linear_cache, activation_cache) we store for computing backward propagation efficiently\n", " activation -- the activation to be used in this layer, stored as a text string: \"sigmoid\" or \"relu\"\n", " \n", " Returns:\n", " dA_prev -- Gradient of the cost with respect to the activation (of the previous layer l-1), same shape as A_prev\n", " dW -- Gradient of the cost with respect to W (current layer l), same shape as W\n", " db -- Gradient of the cost with respect to b (current layer l), same shape as b\n", " \"\"\"\n", " linear_cache, activation_cache = cache\n", " \n", " if activation == \"relu\":\n", " dZ = relu_backward(dA, activation_cache)\n", " dA_prev, dW, db = linear_backward(dZ, linear_cache)\n", " \n", " elif activation == \"sigmoid\":\n", " dZ = sigmoid_backward(dA, activation_cache)\n", " dA_prev, dW, db = linear_backward(dZ, linear_cache)\n", " \n", " return dA_prev, dW, db"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["- 3.4.3 L-Model Backward\n", " \n", "
**Figure 5** : Backward pass
"], "cell_type": "markdown"}, {"metadata": {}, "source": ["np.minimum(dev_labels, 0.999)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {"collapsed": true}, "source": ["def L_model_backward(AL, Y, caches):\n", " \"\"\"\n", " Implement the backward propagation for the [LINEAR->RELU] * (L-1) -> LINEAR -> SIGMOID group\n", " \n", " Arguments:\n", " AL -- probability vector, output of the forward propagation (L_model_forward())\n", " Y -- true \"label\" vector (containing 0 if non-cat, 1 if cat)\n", " caches -- list of caches containing:\n", " every cache of linear_activation_forward() with \"relu\" (there are (L-1) or them, indexes from 0 to L-2)\n", " the cache of linear_activation_forward() with \"sigmoid\" (there is one, index L-1)\n", " \n", " Returns:\n", " grads -- A dictionary with the gradients\n", " grads[\"dA\" + str(l)] = ... \n", " grads[\"dW\" + str(l)] = ...\n", " grads[\"db\" + str(l)] = ... \n", " \"\"\"\n", " grads = {}\n", " L = len(caches) # the number of layers\n", " m = AL.shape[1]\n", " Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL\n", " \n", " # Initializing the backpropagation\n", " #print(25, np.divide(Y, AL))\n", " #print(26, np.divide(1 - Y, 1 - AL))\n", " #print(\"26\",\"before L_model_backward\", \"AL isnan location\", np.argwhere(np.isnan(AL)))\n", " dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n", " #print(\"28\",\"after L_model_backward\", \"dAL isnan location\", np.argwhere(np.isnan(dAL)), np.take(AL, np.argwhere(np.isnan(dAL))))\n", " #print(AL.shape, dAL.shape)\n", " #print(\"isnan dAL:\", dAL[0,734], \"isnan AL:\", AL[0,734])\n", " \n", " #print(\"28\", dAL)\n", " \n", " # Lth layer (SIGMOID -> LINEAR) gradients. Inputs: \"AL, Y, caches\". Outputs: \"grads[\"dAL\"], grads[\"dWL\"], grads[\"dbL\"]\n", " current_cache = caches[L-1]\n", " grads[\"dA\" + str(L)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] = linear_activation_backward(dAL, current_cache, activation = \"sigmoid\")\n", " \n", " for l in reversed(range(L-1)):\n", " # lth layer: (RELU -> LINEAR) gradients.\n", " current_cache = caches[l]\n", " dA_prev_temp, dW_temp, db_temp = linear_activation_backward(grads[\"dA\" + str(l + 2)], current_cache, activation = \"relu\")\n", " grads[\"dA\" + str(l + 1)] = dA_prev_temp\n", " grads[\"dW\" + str(l + 1)] = dW_temp\n", " grads[\"db\" + str(l + 1)] = db_temp\n", "\n", " \n", " return grads"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["** 3.5 Finally update the parameters **\n", "\n", "update the parameters of the model, using gradient descent: \n", "\n", "$$ W^{[l]} = W^{[l]} - \\alpha \\text{ } dW^{[l]} \\tag{16}$$\n", "$$ b^{[l]} = b^{[l]} - \\alpha \\text{ } db^{[l]} \\tag{17}$$\n", "\n", "where $\\alpha$ is the learning rate. After computing the updated parameters, store them in the parameters dictionary. "], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["def update_parameters(parameters, grads, learning_rate):\n", " \"\"\"\n", " Update parameters using gradient descent\n", " \n", " Arguments:\n", " parameters -- python dictionary containing your parameters \n", " grads -- python dictionary containing your gradients, output of L_model_backward\n", " \n", " Returns:\n", " parameters -- python dictionary containing your updated parameters \n", " parameters[\"W\" + str(l)] = ... \n", " parameters[\"b\" + str(l)] = ...\n", " \"\"\"\n", " \n", " L = len(parameters) // 2 # number of layers in the neural network\n", "\n", " # Update rule for each parameter. Use a for loop.\n", " for l in range(L):\n", " #print(\"before update\", \"W\"+str(l+1), parameters[\"W\" + str(l+1)], \"dW\" + str(l+1), grads[\"dW\" + str(l+1)])\n", " parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * grads[\"dW\" + str(l+1)]\n", " parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * grads[\"db\" + str(l+1)]\n", " #print(\"after update\", \"W\"+str(l+1), parameters[\"W\" + str(l+1)], \"dW\" + str(l+1), grads[\"dW\" + str(l+1)])\n", " \n", " return parameters"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["** 3.5.1 Update parameters with Momentum **\n", "\n", "Because mini-batch gradient descent makes a parameter update after seeing just a subset of examples, the direction of the update has some variance, and so the path taken by mini-batch gradient descent will \"oscillate\" toward convergence. Using momentum can reduce these oscillations. \n", "\n", "Momentum takes into account the past gradients to smooth out the update. We will store the 'direction' of the previous gradients in the variable $v$. Formally, this will be the exponentially weighted average of the gradient on previous steps. You can also think of $v$ as the \"velocity\" of a ball rolling downhill, building up speed (and momentum) according to the direction of the gradient/slope of the hill. \n", "\n", "\n", "
**Figure 3**: The red arrows shows the direction taken by one step of mini-batch gradient descent with momentum. The blue points show the direction of the gradient (with respect to the current mini-batch) on each step. Rather than just following the gradient, we let the gradient influence $v$ and then take a step in the direction of $v$.
\n", " \n", "- Initialize the velocity"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["def initialize_velocity(parameters):\n", " \"\"\"\n", " Initializes the velocity as a python dictionary with:\n", " - keys: \"dW1\", \"db1\", ..., \"dWL\", \"dbL\" \n", " - values: numpy arrays of zeros of the same shape as the corresponding gradients/parameters.\n", " Arguments:\n", " parameters -- python dictionary containing your parameters.\n", " parameters['W' + str(l)] = Wl\n", " parameters['b' + str(l)] = bl\n", " \n", " Returns:\n", " v -- python dictionary containing the current velocity.\n", " v['dW' + str(l)] = velocity of dWl\n", " v['db' + str(l)] = velocity of dbl\n", " \"\"\"\n", " \n", " L = len(parameters) // 2 # number of layers in the neural networks\n", " v = {}\n", " \n", " # Initialize velocity\n", " for l in range(L):\n", " v[\"dW\" + str(l+1)] = np.zeros(parameters[\"W\" + str(l+1)].shape)\n", " v[\"db\" + str(l+1)] = np.zeros(parameters[\"b\" + str(l+1)].shape)\n", " \n", " return v"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["- update the parameters with momentum.\n", "The momentum update rule is, for $l = 1, ..., L$: \n", "\n", "$$ \\begin{cases}\n", "v_{dW^{[l]}} = \\beta v_{dW^{[l]}} + (1 - \\beta) dW^{[l]} \\\\\n", "W^{[l]} = W^{[l]} - \\alpha v_{dW^{[l]}}\n", "\\end{cases}\\tag{3}$$\n", "\n", "$$\\begin{cases}\n", "v_{db^{[l]}} = \\beta v_{db^{[l]}} + (1 - \\beta) db^{[l]} \\\\\n", "b^{[l]} = b^{[l]} - \\alpha v_{db^{[l]}} \n", "\\end{cases}\\tag{4}$$\n", "\n", "where L is the number of layers, $\\beta$ is the momentum and $\\alpha$ is the learning rate. All parameters should be stored in the `parameters` dictionary. Note that the iterator `l` starts at 0 in the `for` loop while the first parameters are $W^{[1]}$ and $b^{[1]}$ (that's a \"one\" on the superscript). So you will need to shift `l` to `l+1` when coding."], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["# update_parameters_with_momentum\n", "\n", "def update_parameters_with_momentum(parameters, grads, v, beta, learning_rate):\n", " \"\"\"\n", " Update parameters using Momentum\n", " \n", " Arguments:\n", " parameters -- python dictionary containing your parameters:\n", " parameters['W' + str(l)] = Wl\n", " parameters['b' + str(l)] = bl\n", " grads -- python dictionary containing your gradients for each parameters:\n", " grads['dW' + str(l)] = dWl\n", " grads['db' + str(l)] = dbl\n", " v -- python dictionary containing the current velocity:\n", " v['dW' + str(l)] = ...\n", " v['db' + str(l)] = ...\n", " beta -- the momentum hyperparameter, scalar\n", " learning_rate -- the learning rate, scalar\n", " \n", " Returns:\n", " parameters -- python dictionary containing your updated parameters \n", " v -- python dictionary containing your updated velocities\n", " \"\"\"\n", "\n", " L = len(parameters) // 2 # number of layers in the neural networks\n", " \n", " # Momentum update for each parameter\n", " for l in range(L):\n", " # compute velocities\n", " v[\"dW\" + str(l+1)] = beta * v[\"dW\" + str(l+1)] + (1 - beta) * grads[\"dW\" + str(l+1)]\n", " v[\"db\" + str(l+1)] = beta * v[\"db\" + str(l+1)] + (1 - beta) * grads[\"db\" + str(l+1)]\n", " # update parameters\n", " parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * v[\"dW\" + str(l+1)]\n", " parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * v[\"db\" + str(l+1)]\n", " \n", " return parameters, v"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["**Note** that:\n", "- The velocity is initialized with zeros. So the algorithm will take a few iterations to \"build up\" velocity and start to take bigger steps.\n", "- If $\\beta = 0$, then this just becomes standard gradient descent without momentum. \n", "\n", "**How do you choose $\\beta$?**\n", "\n", "- The larger the momentum $\\beta$ is, the smoother the update because the more we take the past gradients into account. But if $\\beta$ is too big, it could also smooth out the updates too much. \n", "- Common values for $\\beta$ range from 0.8 to 0.999. If you don't feel inclined to tune this, $\\beta = 0.9$ is often a reasonable default. \n", "- Tuning the optimal $\\beta$ for your model might need trying several values to see what works best in term of reducing the value of the cost function $J$. "], "cell_type": "markdown"}, {"metadata": {}, "source": ["\n", "**What you should remember**:\n", "- Momentum takes past gradients into account to smooth out the steps of gradient descent. It can be applied with batch gradient descent, mini-batch gradient descent or stochastic gradient descent.\n", "- You have to tune a momentum hyperparameter $\\beta$ and a learning rate $\\alpha$."], "cell_type": "markdown"}, {"metadata": {}, "source": ["** 3.5.2 - Update parameters with Adam **\n", "\n", "Adam is one of the most effective optimization algorithms for training neural networks. It combines ideas from RMSProp (described in lecture) and Momentum. \n", "\n", "**How does Adam work?**\n", "1. It calculates an exponentially weighted average of past gradients, and stores it in variables $v$ (before bias correction) and $v^{corrected}$ (with bias correction). \n", "2. It calculates an exponentially weighted average of the squares of the past gradients, and stores it in variables $s$ (before bias correction) and $s^{corrected}$ (with bias correction). \n", "3. It updates parameters in a direction based on combining information from \"1\" and \"2\".\n", "\n", "The update rule is, for $l = 1, ..., L$: \n", "\n", "$$\\begin{cases}\n", "v_{dW^{[l]}} = \\beta_1 v_{dW^{[l]}} + (1 - \\beta_1) \\frac{\\partial \\mathcal{J} }{ \\partial W^{[l]} } \\\\\n", "v^{corrected}_{dW^{[l]}} = \\frac{v_{dW^{[l]}}}{1 - (\\beta_1)^t} \\\\\n", "s_{dW^{[l]}} = \\beta_2 s_{dW^{[l]}} + (1 - \\beta_2) (\\frac{\\partial \\mathcal{J} }{\\partial W^{[l]} })^2 \\\\\n", "s^{corrected}_{dW^{[l]}} = \\frac{s_{dW^{[l]}}}{1 - (\\beta_1)^t} \\\\\n", "W^{[l]} = W^{[l]} - \\alpha \\frac{v^{corrected}_{dW^{[l]}}}{\\sqrt{s^{corrected}_{dW^{[l]}}} + \\varepsilon}\n", "\\end{cases}$$\n", "where:\n", "- t counts the number of steps taken of Adam \n", "- L is the number of layers\n", "- $\\beta_1$ and $\\beta_2$ are hyperparameters that control the two exponentially weighted averages. \n", "- $\\alpha$ is the learning rate\n", "- $\\varepsilon$ is a very small number to avoid dividing by zero\n", "\n", "As usual, we will store all parameters in the `parameters` dictionary "], "cell_type": "markdown"}, {"metadata": {}, "source": ["* Initialize the Adam variables $v, s$ which keep track of the past information."], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["# GRADED FUNCTION: initialize_adam\n", "\n", "def initialize_adam(parameters) :\n", " \"\"\"\n", " Initializes v and s as two python dictionaries with:\n", " - keys: \"dW1\", \"db1\", ..., \"dWL\", \"dbL\" \n", " - values: numpy arrays of zeros of the same shape as the corresponding gradients/parameters.\n", " \n", " Arguments:\n", " parameters -- python dictionary containing your parameters.\n", " parameters[\"W\" + str(l)] = Wl\n", " parameters[\"b\" + str(l)] = bl\n", " \n", " Returns: \n", " v -- python dictionary that will contain the exponentially weighted average of the gradient.\n", " v[\"dW\" + str(l)] = ...\n", " v[\"db\" + str(l)] = ...\n", " s -- python dictionary that will contain the exponentially weighted average of the squared gradient.\n", " s[\"dW\" + str(l)] = ...\n", " s[\"db\" + str(l)] = ...\n", "\n", " \"\"\"\n", " \n", " L = len(parameters) // 2 # number of layers in the neural networks\n", " v = {}\n", " s = {}\n", " \n", " # Initialize v, s. Input: \"parameters\". Outputs: \"v, s\".\n", " for l in range(L):\n", " v[\"dW\" + str(l+1)] = np.zeros(parameters[\"W\" + str(l+1)].shape)\n", " v[\"db\" + str(l+1)] = np.zeros(parameters[\"b\" + str(l+1)].shape)\n", " s[\"dW\" + str(l+1)] = np.zeros(parameters[\"W\" + str(l+1)].shape)\n", " s[\"db\" + str(l+1)] = np.zeros(parameters[\"b\" + str(l+1)].shape)\n", " \n", " return v, s"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["* Implement the parameters update with Adam. Recall the general update rule is, for $l = 1, ..., L$: \n", "\n", "$$\\begin{cases}\n", "v_{W^{[l]}} = \\beta_1 v_{W^{[l]}} + (1 - \\beta_1) \\frac{\\partial J }{ \\partial W^{[l]} } \\\\\n", "v^{corrected}_{W^{[l]}} = \\frac{v_{W^{[l]}}}{1 - (\\beta_1)^t} \\\\\n", "s_{W^{[l]}} = \\beta_2 s_{W^{[l]}} + (1 - \\beta_2) (\\frac{\\partial J }{\\partial W^{[l]} })^2 \\\\\n", "s^{corrected}_{W^{[l]}} = \\frac{s_{W^{[l]}}}{1 - (\\beta_2)^t} \\\\\n", "W^{[l]} = W^{[l]} - \\alpha \\frac{v^{corrected}_{W^{[l]}}}{\\sqrt{s^{corrected}_{W^{[l]}}}+\\varepsilon}\n", "\\end{cases}$$\n", "\n", "\n", "**Note** that the iterator `l` starts at 0 in the `for` loop while the first parameters are $W^{[1]}$ and $b^{[1]}$. You need to shift `l` to `l+1` when coding."], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["# GRADED FUNCTION: update_parameters_with_adam\n", "\n", "def update_parameters_with_adam(parameters, grads, v, s, t, learning_rate = 0.01,\n", " beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8):\n", " \"\"\"\n", " Update parameters using Adam\n", " \n", " Arguments:\n", " parameters -- python dictionary containing your parameters:\n", " parameters['W' + str(l)] = Wl\n", " parameters['b' + str(l)] = bl\n", " grads -- python dictionary containing your gradients for each parameters:\n", " grads['dW' + str(l)] = dWl\n", " grads['db' + str(l)] = dbl\n", " v -- Adam variable, moving average of the first gradient, python dictionary\n", " s -- Adam variable, moving average of the squared gradient, python dictionary\n", " learning_rate -- the learning rate, scalar.\n", " beta1 -- Exponential decay hyperparameter for the first moment estimates \n", " beta2 -- Exponential decay hyperparameter for the second moment estimates \n", " epsilon -- hyperparameter preventing division by zero in Adam updates\n", "\n", " Returns:\n", " parameters -- python dictionary containing your updated parameters \n", " v -- Adam variable, moving average of the first gradient, python dictionary\n", " s -- Adam variable, moving average of the squared gradient, python dictionary\n", " \"\"\"\n", " \n", " L = len(parameters) // 2 # number of layers in the neural networks\n", " v_corrected = {} # Initializing first moment estimate, python dictionary\n", " s_corrected = {} # Initializing second moment estimate, python dictionary\n", " \n", " # Perform Adam update on all parameters\n", " for l in range(L):\n", " #print(\"t:\", t)\n", " # Moving average of the gradients. Inputs: \"v, grads, beta1\". Output: \"v\".\n", " v[\"dW\" + str(l+1)] = beta1 * v[\"dW\" + str(l+1)] + (1 - beta1) * grads[\"dW\" + str(l+1)]\n", " v[\"db\" + str(l+1)] = beta1 * v[\"db\" + str(l+1)] + (1 - beta1) * grads[\"db\" + str(l+1)]\n", "\n", " # Compute bias-corrected first moment estimate. Inputs: \"v, beta1, t\". Output: \"v_corrected\".\n", " v_corrected[\"dW\" + str(l+1)] = v[\"dW\" + str(l+1)]/(1 - np.power(beta1, 2))\n", " v_corrected[\"db\" + str(l+1)] = v[\"db\" + str(l+1)]/(1 - np.power(beta1, 2))\n", "\n", " # Moving average of the squared gradients. Inputs: \"s, grads, beta2\". Output: \"s\".\n", " s[\"dW\" + str(l+1)] = beta2 * s[\"dW\" + str(l+1)] + (1 - beta2) * np.power(grads[\"dW\" + str(l+1)], 2)\n", " s[\"db\" + str(l+1)] = beta2 * s[\"db\" + str(l+1)] + (1 - beta2) * np.power(grads[\"db\" + str(l+1)], 2)\n", " #print(\"46\", grads[\"dW\" + str(l+1)])\n", " #print(\"47\", s[\"dW\" + str(l+1)])\n", " \n", " # Compute bias-corrected second raw moment estimate. Inputs: \"s, beta2, t\". Output: \"s_corrected\".\n", " \n", " \n", " s_corrected[\"dW\" + str(l+1)] = s[\"dW\" + str(l+1)]/(1 - np.power(beta2, t))\n", " s_corrected[\"db\" + str(l+1)] = s[\"db\" + str(l+1)]/(1 - np.power(beta2, t))\n", " #print(\"54\", s_corrected[\"dW\" + str(l+1)])\n", "\n", " # Update parameters. Inputs: \"parameters, learning_rate, v_corrected, s_corrected, epsilon\". Output: \"parameters\".\n", " #parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * (v_corrected[\"dW\" + str(l+1)]/(epsilon + np.power(s_corrected[\"dW\" + str(l+1)], 0.5))) ##\n", " #parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * (v_corrected[\"db\" + str(l+1)]/(epsilon + np.power(s_corrected[\"db\" + str(l+1)], 0.5))) ###\n", " #print((np.sqrt(s_corrected[\"dW\" + str(l+1)])))\n", " #print(learning_rate * v_corrected[\"dW\" + str(l+1)] / (np.sqrt(s_corrected[\"dW\" + str(l+1)]) + epsilon))\n", " parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * v_corrected[\"dW\" + str(l+1)] / (np.sqrt(s_corrected[\"dW\" + str(l+1)]) + epsilon) \n", " parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * v_corrected[\"db\" + str(l+1)] / (np.sqrt(s_corrected[\"db\" + str(l+1)]) + epsilon)\n", " return parameters, v, s"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["### 4. Merge all functions into a model"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["layers_dims = [49152, 5, 20, 1]"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {"collapsed": true}, "source": ["def L_layer_model(X, Y, layers_dims, parameters=None, learning_rate = 0.075,\n", " num_epochs = 3000, optimizer=\"adam\", mini_batch_size = 64,\n", " beta = 0.9, beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8, print_cost=False):\n", " \"\"\"\n", " Implements a L-layer neural network: [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID.\n", " \n", " Arguments:\n", " X -- data, numpy array of shape (number of examples, num_px * num_px * 3)\n", " Y -- true \"label\" vector (containing 0 if cat, 1 if non-cat), of shape (1, number of examples)\n", " layers_dims -- list containing the input size and each layer size, of length (number of layers + 1).\n", " parameters -- An initial value of the weights, default is None.Parameters initialited by this value.\n", " learning_rate -- learning rate of the gradient descent update rule\n", " num_epochs -- number of epochs\n", " print_cost -- if True, it prints the cost every 100 steps\n", " mini_batch_size -- the size of a mini batch\n", " beta -- Momentum hyperparameter\n", " beta1 -- Exponential decay hyperparameter for the past gradients estimates \n", " beta2 -- Exponential decay hyperparameter for the past squared gradients estimates \n", " epsilon -- hyperparameter preventing division by zero in Adam updates\n", " \n", " Returns:\n", " parameters -- parameters learnt by the model. They can then be used to predict.\n", " \"\"\"\n", " \n", " L = len(layers_dims) # number of layers in the neural networks\n", " costs = [] # keep track of cost\n", " t = 0 # initializing the counter required for Adam update\n", " \n", " # Parameters initialization.\n", " ## Parameters: python dictionary containing your parameters \"W1\", \"b1\", ..., \"WL\", \"bL\"\n", " if not parameters:\n", " parameters = initialize_parameters_deep(layers_dims)\n", " \n", " # Initialize the optimizer\n", " if optimizer == \"gd\":\n", " pass # no initialization required for gradient descent\n", " elif optimizer == \"momentum\":\n", " v = initialize_velocity(parameters)\n", " elif optimizer == \"adam\":\n", " v, s = initialize_adam(parameters)\n", " \n", " # Loop (gradient descent)\n", " for i in range(num_epochs):\n", "\n", " # Forward propagation: [LINEAR -> RELU]*(L-1) -> LINEAR -> SIGMOID.\n", " '''\n", " AL -- last post-activation value\n", " caches -- list of caches containing:\n", " every cache of linear_activation_forward('relu') (there are L-1 of them, indexed from 0 to L-2)\n", " the cache of linear_activation_forward('sigmoid') (there is one, indexed L-1)\n", " '''\n", " minibatches = random_mini_batches(X, Y, mini_batch_size, shuffle=True)\n", " \n", " for minibatch in minibatches:\n", " #print(\"55 isnan in X\",np.argwhere(np.isnan(X)))\n", " (minibatch_X, minibatch_Y) = minibatch\n", " AL, caches = L_model_forward(minibatch_X, parameters)\n", " #print(\"57 isnan in AL\",np.argwhere(np.isnan(AL)))\n", " #print(\"58 caches\",caches)\n", " # Compute cost.\n", " cost = compute_cost(AL, minibatch_Y)\n", " \n", " # Backward propagation.\n", " grads = L_model_backward(AL, minibatch_Y, caches)\n", " #print(\"64 grads\",grads)\n", " #print(\"63\", grads)\n", " \n", " # Update parameters.\n", " if optimizer == \"gd\":\n", " parameters = update_parameters(parameters, grads, learning_rate)\n", " elif optimizer == \"momentum\":\n", " parameters, v = update_parameters_with_momentum(parameters, grads, v, beta, learning_rate)\n", " elif optimizer == \"adam\":\n", " t = t + 1 # Adam counter\n", " parameters, v, s = update_parameters_with_adam(parameters, grads, v, s,\n", " t, learning_rate, beta1, beta2, epsilon)\n", " \n", " \n", " # Print the cost every 100 training example\n", " if print_cost and i % 100 == 0:\n", " print (\"Cost after epoch %i: %f\" %(i, cost))\n", " if print_cost and i % 10 == 0:\n", " costs.append(cost)\n", " \n", " # plot the cost\n", " plt.plot(np.squeeze(costs))\n", " plt.ylabel('cost')\n", " plt.xlabel('epoches (per tens)')\n", " plt.title(\"Learning rate =\" + str(learning_rate))\n", " plt.show()\n", " \n", " return parameters, costs"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {"scrolled": false}, "source": ["trained_parameters, costs = L_layer_model(train_images, train_labels, layers_dims, parameters=None, learning_rate = 0.075,\n", " num_epochs = 1001, optimizer=\"gd\", mini_batch_size = 64,\n", " beta = 0.9, beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8, print_cost=True)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {"collapsed": true}, "source": ["def predict(X, y, parameters):\n", " \"\"\"\n", " This function is used to predict the results of a L-layer neural network.\n", " \n", " Arguments:\n", " X -- data set of examples you would like to label\n", " parameters -- parameters of the trained model\n", " \n", " Returns:\n", " p -- predictions for the given dataset X\n", " \"\"\"\n", " \n", " m = X.shape[1]\n", " n = len(parameters) // 2 # number of layers in the neural network\n", " p = np.zeros((1,m))\n", " \n", " # Forward propagation\n", " probas, caches = L_model_forward(X, parameters)\n", "\n", " \n", " # convert probas to 0/1 predictions\n", " for i in range(0, probas.shape[1]):\n", " if probas[0,i] > 0.5:\n", " p[0,i] = 1\n", " else:\n", " p[0,i] = 0\n", " \n", " #print results\n", " #print (\"predictions: \" + str(p))\n", " #print (\"true labels: \" + str(y))\n", " print(\"Accuracy: \" + str(np.sum((p == y)/m)))\n", " \n", " return p"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["pred_train = predict(train_images, train_labels, trained_parameters)\n", "pred_test = predict(dev_images, dev_labels, trained_parameters)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["### 5. Visualizing and Analysis"], "cell_type": "markdown"}, {"metadata": {"collapsed": true}, "source": ["## First, let's take a look at some images the L-layer model labeled incorrectly. This will show a few mislabeled images. \n", "def print_mislabeled_images(classes, X, y, p):\n", " \"\"\"\n", " Plots images where predictions and truth were different.\n", " X -- dataset\n", " y -- true labels\n", " p -- predictions\n", " \"\"\"\n", " a = p + y\n", " mislabeled_indices = np.asarray(np.where(a == 1))\n", " plt.rcParams['figure.figsize'] = (40.0, 40.0) # set default size of plots\n", " num_images = len(mislabeled_indices[0])\n", " for i in range(num_images):\n", " index = mislabeled_indices[1][i]\n", " \n", " plt.subplot(2, num_images, i + 1)\n", " plt.imshow(X[:,index].reshape(IMAGE_WIDTH,IMAGE_HEIGHT,3), interpolation='nearest')\n", " plt.axis('off')\n", " plt.title(\"Prediction: \" + classes[int(p[0,index])].decode(\"utf-8\") + \" \\n Class: \" + classes[y[0,index]].decode(\"utf-8\"))"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {"collapsed": true}, "source": ["classes = np.array([b'cat',b'dog'])"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["print_mislabeled_images(classes, dev_images, dev_labels, pred_test)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["### 6. LAB"], "cell_type": "markdown"}, {"metadata": {}, "source": ["- 6.1 Different optimization methods"], "cell_type": "markdown"}, {"metadata": {}, "source": ["images, labels = load_images(file_count=100)\n", "dev_images, train_images = split_data(images)\n", "dev_labels, train_labels = split_data(labels)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["trained_parameters, costs = L_layer_model(train_images, train_labels, layers_dims, parameters=None,\n", " num_epochs = 1001, optimizer=\"gd\", mini_batch_size = 64, learning_rate=0.0075,\n", " beta = 0.9, beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8, print_cost=True)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["pred_train = predict(train_images, train_labels, trained_parameters)\n", "pred_test = predict(dev_images, dev_labels, trained_parameters)\n", "print_mislabeled_images(classes, dev_images, dev_labels, pred_test)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["trained_parameters, costs = L_layer_model(train_images, train_labels, layers_dims, parameters=None, learning_rate = 0.0075,\n", " num_epochs = 1001, optimizer=\"momentum\", mini_batch_size = 64,\n", " beta = 0.9, beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8, print_cost=True)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["pred_train = predict(train_images, train_labels, trained_parameters)\n", "pred_test = predict(dev_images, dev_labels, trained_parameters)\n", "print_mislabeled_images(classes, dev_images, dev_labels, pred_test)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["trained_parameters, costs = L_layer_model(train_images, train_labels, layers_dims, parameters=None, learning_rate = 0.0075,\n", " num_epochs = 1001, optimizer=\"adam\", mini_batch_size = 64,\n", " beta = 0.9, beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8, print_cost=True)"], "cell_type": "code", "execution_count": null, "outputs": []}, {"metadata": {}, "source": ["pred_train = predict(train_images, train_labels, trained_parameters)\n", "pred_test = predict(dev_images, dev_labels, trained_parameters)\n", "print_mislabeled_images(classes, dev_images, dev_labels, pred_test)"], "cell_type": "code", "execution_count": null, "outputs": []}], "metadata": {"kernelspec": {"name": "python3", "language": "python", "display_name": "Python 3"}, "language_info": {"name": "python", "nbconvert_exporter": "python", "version": "3.5.2", "pygments_lexer": "ipython3", "mimetype": "text/x-python", "codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py"}}, "nbformat_minor": 1, "nbformat": 4}