{"nbformat": 4, "nbformat_minor": 1, "metadata": {"language_info": {"nbconvert_exporter": "python", "pygments_lexer": "ipython3", "mimetype": "text/x-python", "name": "python", "codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "version": "3.6.1"}, "kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}}, "cells": [{"execution_count": null, "outputs": [], "metadata": {"_uuid": "b5deb0d5a0abc75e0dc14f27c180ace5dcdb1296", "collapsed": true, "_cell_guid": "6435dd55-97a5-48cc-86ce-b3c1c9494ade"}, "cell_type": "code", "source": ["%matplotlib inline\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "import cv2"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "c0cc89d65aa051fdd178a1fdc5434868f5c66d17", "collapsed": true, "_cell_guid": "95997dd5-9174-4265-80e3-3b18593759bf"}, "cell_type": "code", "source": ["# reading the training and testing dataset from csv files\n", "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "27d19deb0f88fe1230005a013adb10776b20c567", "scrolled": true, "_cell_guid": "818b3f10-94c4-4add-8b34-275a23e07ee8"}, "cell_type": "code", "source": ["train.head()"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "e868f11ee7ef179b4080280260793cfd089e8c4e", "_cell_guid": "4025d53c-7d84-4dfe-a3a0-cccc002e00aa"}, "cell_type": "code", "source": ["test.head()"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "31c38a3ef4eb94067cc5613b152ef7597efae80a", "collapsed": true, "_cell_guid": "f015a5da-0431-4326-bf01-d03df6dcc761"}, "cell_type": "code", "source": ["# labels of training features\n", "train_y = train[\"label\"]"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "6e76214ca83dc9b448e7bf4b45bde0a169753514", "_cell_guid": "396e5e29-5f1b-42aa-b741-9308e3b8dcc1"}, "cell_type": "code", "source": ["train_y.head()"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "abbd2eb487e5e8dc9dd93c1c9ef88b4f66c7cd68", "collapsed": true, "_cell_guid": "b88e4238-168a-497c-8294-806bcb9ecef9"}, "cell_type": "code", "source": ["# finding the training freature\n", "train_x = train.iloc[:,1:785]"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "9940d71520386721680f0bb5e93c83fb0ed2232b", "_cell_guid": "f37d118a-83de-4f32-80f5-fb328945e9b0"}, "cell_type": "code", "source": ["train_x.head()"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "9ee6272143f773e8b87fd21a4f8929b25216ca98", "_cell_guid": "0abe160b-0682-4ef6-a39c-68d3cdd04d78"}, "cell_type": "code", "source": ["# Explore the dataset\n", "m_train = train.shape[0]\n", "num_px = test.shape[1]\n", "m_test = test.shape[0]\n", "print(\"Number of training examples: \" + str(m_train))\n", "print(\"Number of testing examples: \"+ str(m_test))\n", "print(\"Size of each image: (\"+ str(int(np.sqrt(num_px))) + \", \" + str(int(np.sqrt(num_px)))+\")\")\n", "print(\"Shape of training features: \" + str(train.shape))\n", "print(\"Shape of testing features: \" + str(test.shape))\n", "print(\"Shape of labels of tarining features: \" + str(train_y.shape))"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "9136b43b03780d3c9d3d35647bf4e4588ed31c2b", "_cell_guid": "3905e94b-62c7-42af-906a-d131ab876d3c"}, "cell_type": "code", "source": ["# Examples of digits\n", "index = 100\n", "plt.figure(1)\n", "plt.imshow(train_x.iloc[index,:].reshape(28,28), cmap = 'gray')\n", "print(\"Corresponding label = \" + str(train_y[index]))"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "cddd84eda8493f770e0b3c2c385b5f479c8ad9ce", "_cell_guid": "a7e53db4-de41-48f7-ace3-20fa73e8f9ff"}, "cell_type": "code", "source": ["# Standardize data to have feature values between 0 and 1\n", "tarin_x = train_x/255\n", "test_x = test/255\n", "\n", "print(\"train_x's shape: \"+str(train_x.shape))\n", "print(\"test_x's shape: \"+str(test_x.shape))"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "8cc523cee2e08e5b786eab8bdb624a95a19e707a", "collapsed": true, "_cell_guid": "cc77b5e0-9a93-496e-84d5-f3e22fe14577"}, "cell_type": "code", "source": ["# initialize parameters\n", "def initialize_parameters(n_x,n_h,n_y):\n", "    \"\"\"\n", "    inputs:\n", "    n_x -- size of input layer\n", "    n_h -- size of hidden layer\n", "    n_y -- size of output layer\n", "    \n", "    outputs:\n", "    parameters\n", "    \"\"\"\n", "    # W1 : weight of matrix of shape (n_h,n_x)\n", "    W1 = np.random.randn(n_h,n_x) * 0.01\n", "    # b1 : bias vector of shape (n_h,1)\n", "    b1 = np.zeros((n_h,1))\n", "    # W2 : weight of matrix of shape (n_y,n_h)\n", "    W2 = np.random.randn(n_y,n_h) * 0.01\n", "    # b2 : bias vector of shape (n_y,1)\n", "    b2 = np.zeros((n_y,1))\n", "    \n", "    parameters = {\"W1\" : W1,\n", "                 \"b1\" : b1,\n", "                 \"W2\" : W2,\n", "                 \"b2\" : b2}\n", "    \n", "    return parameters"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "cf8496a177ed4acd732f38304a174987b8ce98ea", "_cell_guid": "bf51a359-df89-442b-b574-0941897d9118"}, "cell_type": "code", "source": ["# initial parameters\n", "parametrs = initialize_parameters(2,2,1)\n", "print(\"W1 = \"+str(parametrs[\"W1\"]))\n", "print(\"b1 = \"+str(parametrs[\"b1\"]))\n", "print(\"W2 = \"+str(parametrs[\"W2\"]))\n", "print(\"b2 = \"+str(parametrs[\"b2\"]))"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "517e518f152d3680d799c330f951ede73daa6698", "collapsed": true, "_cell_guid": "5528a8c1-576d-4cc8-b27a-547c86cd5535"}, "cell_type": "code", "source": ["def initialize_parameters_deep(layer_dims):\n", "    \"\"\"\n", "    inputs:\n", "    layer_dims: list of the dimention of each layer\n", "    \n", "    outputs:\n", "    parameters: containing the weight and bias parameters of deep neural networks\n", "    Wl : weight matrix of shape (layer_dims[l],layer_dims[l-1])\n", "    bl : bias vector of shape (layer_dims[l],1)\n", "    \"\"\"\n", "    parameters = {}\n", "    # number of layers in the network\n", "    L = len(layer_dims)\n", "    \n", "    for l in range(1,L):\n", "        parameters[\"W\" + str(l)] = np.random.randn(layer_dims[l],layer_dims[l-1]) * 0.001\n", "        parameters[\"b\" + str(l)] = np.zeros((layer_dims[l],1))\n", "        \n", "    return parameters   "]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "625b427c4070ea7e7007e500f2039e4be440da80", "_cell_guid": "6427cfc8-f1b1-4bc8-9ae6-445c8362bfce"}, "cell_type": "code", "source": ["# initial parameters of deep neural nets\n", "parametrs = initialize_parameters_deep([6,4,3])\n", "print(\"W1 = \"+str(parametrs[\"W1\"]))\n", "print(\"b1 = \"+str(parametrs[\"b1\"]))\n", "print(\"W2 = \"+str(parametrs[\"W2\"]))\n", "print(\"b2 = \"+str(parametrs[\"b2\"]))"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "c481db34d95f9dc4fcd39937f1acd42b6da48f69", "collapsed": true, "_cell_guid": "6077f0e1-8f43-4ca5-82bf-f213e69b7640"}, "cell_type": "code", "source": ["# forward propagation Module\n", "# building the linear part of NN\n", "def linear_forward(A,W,b):\n", "    '''\n", "    inputs:\n", "    A --- Activation from previous layer\n", "    W,b --- parameters of NN\n", "    \n", "    outputs:\n", "    Z --- linear equation or input for the activation layer\n", "    cache --- stored A, W and b\n", "    '''\n", "    Z = np.matmul(W,A) + b\n", "    cache = (A, W, b)\n", "    return Z, cache"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "1a0a04f555d9fb90e38f89eceb218b2f3b600806", "collapsed": true, "_cell_guid": "24598976-8dcd-464a-b357-1a14a02c7c7a"}, "cell_type": "code", "source": ["# sigmoid function \n", "def sigmoid(Z):\n", "    '''\n", "    Z : input value, vector or matrix\n", "    \n", "    A : sigmoid of Z\n", "    '''\n", "    A = 1/(1+np.exp(-Z))\n", "    cache = Z\n", "    return A, cache"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "3124b30df7e879f3f4ba54bf6a3106b0dfe25822", "collapsed": true, "_cell_guid": "081e9232-afec-4e5a-adf8-88e67d939a64"}, "cell_type": "code", "source": ["# derivative of sigmoid function\n", "def sigmoid_backward(dA, activation_cache):\n", "    #print(\"dA: \"+str(dA))\n", "    Z = activation_cache\n", "    #print(\"Z: \"+str(Z))\n", "    A,cache = sigmoid(Z)\n", "    dSig = np.multiply(A, (1-A))\n", "    #print(\"dSig: \"+str(dSig))\n", "    dZ = np.multiply(dA,dSig)\n", "    #print(\"dZ: \"+str(dZ))\n", "    return dZ"]}, {"source": ["def softmax(Z):\n", "    exps = np.exp(Z)\n", "    A = exps/np.sum(exps)\n", "    cache = Z\n", "    return A, cache"], "metadata": {"_uuid": "13a7e5b2333b5c6f01bf04ea2099e26fe65e4167", "collapsed": true, "_cell_guid": "af4c38ee-ad97-4690-9967-5209c97a2bf7"}, "cell_type": "markdown"}, {"source": ["def softmax_backward(dA, activation_cache):\n", "    Z = activation_cache\n", "    A,cache = softmax(Z)\n", "    dSoft = np.multiply(A,(1-A))\n", "    dZ = np.multiply(dA,dSoft)\n", "    return dZ"], "metadata": {"_uuid": "b7a188250bd0d56d4d647b0f557da7ae410ab486", "collapsed": true, "_cell_guid": "ad0021a8-52c2-4700-9524-3a6b4b007d7a"}, "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "0df0916d331afcbf9975fed2347383418040347c", "collapsed": true, "_cell_guid": "06a712bf-913b-498e-a67d-71302b858661"}, "cell_type": "code", "source": ["# ReLU function\n", "def relu(Z):\n", "    A = Z * (Z>0)\n", "    cache = Z\n", "    return A, cache\n"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "d62f50050804b6a8dd40b5fbbda57fa40c5d20ee", "collapsed": true, "_cell_guid": "f078f7f7-4498-4d9d-b706-4a0467132a2f"}, "cell_type": "code", "source": ["# Derivative of ReLU function\n", "def relu_backward(dA, activation_cache):\n", "    Z = activation_cache\n", "    dRelu = (Z > 0)\n", "    dRelu = dRelu.astype(float)\n", "    dZ = np.multiply(dA,dRelu)\n", "    return dZ"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "9afdd1f7b259e4d759412ebcf63dbbcd9d564758", "collapsed": true, "_cell_guid": "7c859f42-981d-4571-9c2e-7509a02caf4e"}, "cell_type": "code", "source": ["# linear activation function\n", "def linear_activation_forward(A_prev, W, b, activation):\n", "    '''\n", "    inputs: \n", "    A_prev, W, b --- As defined in the linear_forward function\n", "    activation  --- activation to be used\n", "    \n", "    outputs:\n", "    A -- output of the activation function\n", "    cache -- containing linear cache and activation cache which will use in backpropagation \n", "            calulation         \n", "   '''\n", "    if activation == \"sigmoid\":\n", "        Z, linear_cache = linear_forward(A_prev,W,b)\n", "        #print(\"Z: \"+str(Z))\n", "        A, activation_cache = sigmoid(Z)\n", "    \n", "    elif activation == \"relu\":\n", "        Z, linear_cache = linear_forward(A_prev,W,b)\n", "        A, activation_cache = relu(Z)\n", "        \n", "    cache = (linear_cache, activation_cache)\n", "    \n", "    return A, cache"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "dd270af9d62e61bf1b2dc0aeb6f9cf1e16920b85", "collapsed": true, "_cell_guid": "c95abffb-e4fc-4c76-b033-ce96ded25a52"}, "cell_type": "code", "source": ["# L-layer linear activation function\n", "def L_model_forward(X, parameters):\n", "    '''\n", "    X --- data\n", "    parameters --- output of initialize_parameters_deep()\n", "    \n", "    AL -- last post activation value\n", "    '''\n", "    caches = []\n", "    A = X\n", "    L = len(parameters)//2\n", "    # implimentation of [Linear ---> RELU * (L-1)]\n", "    for l in range(1,L):\n", "        A_prev = A\n", "        A, cache = linear_activation_forward(A_prev, parameters[\"W\" + str(l)], parameters[\"b\" + str(l)],\"relu\")\n", "        caches.append(cache)\n", "    # implimentation of Linear ---> sigmoid\n", "    AL, cache = linear_activation_forward(A, parameters[\"W\" + str(L)], parameters[\"b\" + str(L)], \"sigmoid\")\n", "    caches.append(cache)\n", "    \n", "    return AL, caches\n", "    "]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "98e790259427a66fec382668824e0e9280645905", "collapsed": true, "_cell_guid": "5954747b-6497-4a51-8968-2c5b4e0ccd58"}, "cell_type": "code", "source": ["def one_hot_matrix(Y, n_y):\n", "    Y = np.vstack(Y)\n", "    Y_ = np.zeros((Y.shape[0],n_y))\n", "    Y_[np.arange(Y.shape[0]),Y.T] = 1\n", "    Y_ = Y_.T\n", "    return Y_\n", "    "]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "21962985681e03c91b41042e45c5b930c53ec994", "scrolled": true, "collapsed": true, "_cell_guid": "4b192fab-7c31-499d-98b2-d7f17a4e473b"}, "cell_type": "code", "source": ["Y = one_hot_matrix(train_y,10)"]}, {"source": ["# computing the cost function\n", "def compute_cost1(AL, Y):\n", "    '''\n", "    AL --- probability vector corresponding to label prediction\n", "    Y --- correct label vector\n", "    \n", "    cost --- entropy cost\n", "    '''\n", "    m = Y.shape[0]\n", "    #print(AL[1,:])\n", "    cost = 0\n", "    for i in range(0,10):\n", "        cost = cost-np.sum(np.multiply(np.log(AL[i,:]),(Y==i)) + np.multiply(np.log(1-AL[i,:]),(1-(Y==i))))/m\n", "    cost = np.squeeze(cost)\n", "    return cost"], "metadata": {"_uuid": "d06afb784209c1cd531d14e6d7c079ef7c92ccfe", "collapsed": true, "_cell_guid": "e82cc49c-4f32-4609-8554-de50381324a4"}, "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "52b73695b26a6544e27de9444663b44213677212", "collapsed": true, "_cell_guid": "56362ed7-ccf8-496b-aae9-a85b9879b126"}, "cell_type": "code", "source": ["def compute_cost(AL, Y):\n", "    m = Y.shape[1]\n", "    cost = -np.sum(np.multiply(np.log(AL),Y)+np.multiply(np.log(1-AL),(1-Y)))/m\n", "    return cost"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "6723d46ae223476bf471d259314f905160a09744", "collapsed": true, "_cell_guid": "7898f742-8d16-48ae-b059-998e175fea17"}, "cell_type": "code", "source": ["# Backward propagation module\n", "# Implimentation of backward propagation for single layer\n", "def linear_backward(dZ, cache):\n", "    A_prev, W, b = cache\n", "    m = A_prev.shape[1]\n", "    dW = np.matmul(dZ,A_prev.T)/m\n", "    db = np.sum(dZ, axis=1,keepdims = True)/m\n", "    dA_prev = np.matmul(W.T,dZ)\n", "    return dA_prev, dW, db\n", "    "]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "de24395bb97d5882b7a0be63b80ddc0567b9ef00", "collapsed": true, "_cell_guid": "9f045621-c2e7-4383-8310-69fd76b62815"}, "cell_type": "code", "source": ["# linear activation backward\n", "def linear_activation_backward(dA, cache, activation):\n", "    linear_cache, activation_cache = cache\n", "    if activation == \"relu\":\n", "        dZ = relu_backward(dA, activation_cache)\n", "        dA_prev, dW, db = linear_backward(dZ, linear_cache)\n", "        \n", "    if activation == \"sigmoid\":\n", "        dZ = sigmoid_backward(dA, activation_cache)\n", "        dA_prev, dW, db = linear_backward(dZ, linear_cache)\n", "    \n", "    return dA_prev, dW, db"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "9b1e0c5b413cb5d883da28415b45b292ee888c9a", "collapsed": true, "_cell_guid": "224f7e1d-f09f-4b98-8015-7d5d4750edb7"}, "cell_type": "code", "source": ["#implimentation of L layer backward activation function\n", "def L_model_backward(AL, Y, caches):\n", "    grads = {}\n", "    L = len(caches)\n", "    m = AL.shape[1]\n", "    Y = Y.reshape(AL.shape)\n", "    dAL = -(np.divide(Y,AL)-np.divide(1-Y,1-AL))\n", "    current_cache = caches[L-1]\n", "    grads[\"dA\"+str(L)], grads[\"dW\"+str(L)],grads[\"db\"+str(L)] = linear_activation_backward(dAL, current_cache, \"sigmoid\")\n", "    \n", "    for l in reversed(range(L-1)):\n", "        current_cache = caches[l]\n", "        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(grads[\"dA\"+str(l+2)],current_cache,\"relu\")\n", "        grads[\"dA\"+str(l+1)] = dA_prev_temp\n", "        grads[\"dW\" + str(l+1)] = dW_temp\n", "        grads[\"db\" + str(l+1)] = db_temp\n", "    return grads"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "f18f45c9c0d43f4a6824cd3bce665d32cd57210f", "collapsed": true, "_cell_guid": "c0dc6643-dab2-4c97-877e-27e811760e85"}, "cell_type": "code", "source": ["# implimentation for parameter updation\n", "def update_parameters(parameters, grads, learning_rate):\n", "    L = len(parameters)//2\n", "    for l in range(L):\n", "        parameters[\"W\"+str(l+1)] = parameters[\"W\"+str(l+1)] - learning_rate * grads[\"dW\"+str(l+1)]\n", "        parameters[\"b\"+str(l+1)] = parameters[\"b\"+str(l+1)] - learning_rate * grads[\"db\"+str(l+1)]\n", "    return parameters"]}, {"source": ["# Two layer model\n", "def two_layer_model(X, Y, layers_dims, learning_rate = 0.0001, num_iteration = 3000):\n", "    costs = []\n", "    grads = {}\n", "    m = X.shape[0]\n", "    (n_x,n_h,n_y) = layers_dims\n", "    parameters = initialize_parameters(n_x, n_h, n_y)\n", "    W1 = parameters[\"W1\"]\n", "    b1 = parameters[\"b1\"]\n", "    W2 = parameters[\"W2\"]\n", "    b2 = parameters[\"b2\"]\n", "    Y_ = Y\n", "    Y = one_hot_matrix(Y,n_y)\n", "    \n", "    for i in range(0,num_iteration):\n", "        A1, cache1 = linear_activation_forward(X, W1, b1, \"relu\")\n", "        A2, cache2 = linear_activation_forward(A1, W2, b2, \"sigmoid\")\n", "        cost = compute_cost(A2, Y)\n", "        #Y = one_hot_matrix(Y_,n_y)\n", "        #cost1 = compute_cost1(A2,Y_)\n", "        #print(cost1,cost)\n", "        # initializing backward propagation\n", "        #print(\"cost1: \"+str(cost1))\n", "        #print(\"cost: \"+str(cost))\n", "        #print(\"A1: \"+str(A1))\n", "        #print(\"A2: \"+str(A2))\n", "        #print(cost1)\n", "        dA2 = -(np.divide(Y,A2) - np.divide((1-Y),(1-A2)))\n", "        #dA2 = A2 - Y\n", "        #print(\"dA2: \"+str(dA2))\n", "        #print(\"cache2 : \"+str(cache2))\n", "        dA1, dW2, db2 = linear_activation_backward(dA2, cache2, \"sigmoid\")\n", "        dA0, dW1, db1 = linear_activation_backward(dA1, cache1, \"relu\")\n", "        \n", "        grads[\"dW1\"] = dW1\n", "        grads[\"db1\"] = db1\n", "        grads[\"dW2\"] = dW2\n", "        grads[\"db2\"] = db2\n", "        parameters = update_parameters(parameters, grads, learning_rate)\n", "        \n", "        W1 = parameters[\"W1\"]\n", "        b1 = parameters[\"b1\"]\n", "        W2 = parameters[\"W2\"]\n", "        b2 = parameters[\"b2\"]\n", "        if i % 100 == 0:\n", "            print(\"Cost after iteration {}: {}\".format(i,np.squeeze(cost)))\n", "        if i%100 == 0:\n", "            costs.append(cost)\n", "            \n", "    # plot the cost\n", "    plt.figure(6)\n", "    plt.plot(np.squeeze(costs))\n", "    plt.ylabel('cost')\n", "    plt.xlabel('iterations (per tens)')\n", "    plt.title(\"Learning rate = \"+str(learning_rate))\n", "    plt.show()\n", "    return parameters"], "metadata": {"_uuid": "651d747f5a24ab90d06a1b368c9c100ba7215491", "collapsed": true, "_cell_guid": "f6cf2e9a-2926-41ca-8d86-151a8ccea87d"}, "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "730e2e5065434d08afa8ffc67bec00d55a1bf6f4", "collapsed": true, "_cell_guid": "91dbecd3-c73d-41d5-bc8c-1c5fa7954011"}, "cell_type": "code", "source": ["# implementation of L layer model\n", "def L_layer_model(X, Y, layers_dims, learning_rate, num_iterations, print_cost):\n", "    costs = []\n", "    # parameters initialization\n", "    parameters = initialize_parameters_deep(layers_dims)\n", "    # one hot matrix conversion\n", "    Y = one_hot_matrix(Y,layers_dims[-1])\n", "    # gradient descent\n", "    for i in range(0, num_iterations):\n", "        # forward propagation\n", "        AL, caches = L_model_forward(X, parameters)\n", "        # cost computation\n", "        cost = compute_cost(AL, Y)\n", "        # backward propagation\n", "        grads = L_model_backward(AL, Y, caches)\n", "        # parameter updation\n", "        parameters = update_parameters(parameters, grads, learning_rate)\n", "        # print the cost every 100 iterations\n", "        if print_cost and i%100 == 0:\n", "            print(\"cost after iteration %i: %f\" %(i,cost))\n", "        if print_cost and i%100 == 0:\n", "            costs.append(cost)\n", "            \n", "   # plot the cost \n", "    plt.figure(7)\n", "    plt.plot(np.squeeze(costs))\n", "    plt.ylabel('cost')\n", "    plt.xlabel('iterations (per tens)')\n", "    plt.title(\"Learning rate = \"+str(learning_rate))\n", "    plt.show()\n", "    return parameters"]}, {"source": ["# Constant for the two layer model\n", "n_x = 784 ; # number of pixels of each image\n", "n_h = 30\n", "n_y = 10; # number of possible digits\n", "#layers_dims = (n_x, n_h, n_y)\n", "#train_y = np.array(train_y)\n", "#parameters = two_layer_model(train_x.T,train_y,layers_dims)"], "metadata": {"_uuid": "6128d9988f55f386d518c9ae758009465c1f2ce1", "collapsed": true, "_cell_guid": "b369740f-9e8f-4e6e-b309-f6d541e03317"}, "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "a4f4dbea706e62299e044632987510a9e8cc67f3", "scrolled": false, "_cell_guid": "d0cef73c-596d-4430-8880-ce7eccc7499e"}, "cell_type": "code", "source": ["# Constant for the L layer model\n", "X = train_x.T\n", "Y = np.array(train_y)\n", "layers_dims = [784, 128, 64, 10]\n", "learning_rate = 0.005\n", "num_iterations = 3000\n", "print_cost = True\n", "parameters = L_layer_model(X, Y, layers_dims, learning_rate, num_iterations, print_cost)"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "b1f624fcbf4160788bd5870eabc7c986ce46f582", "_cell_guid": "f0e84c38-323e-4763-99df-75c06425535c"}, "cell_type": "code", "source": ["print(parameters)"]}, {"source": ["def predict(X, y, parameters):\n", "    m = X.shape[1]\n", "    \n", "    W1 = parameters[\"W1\"]\n", "    b1 = parameters[\"b1\"]\n", "    W2 = parameters[\"W2\"]\n", "    b2 = parameters[\"b2\"]\n", "    \n", "    y = one_hot_matrix(y,b2.shape[0])\n", "    y_prediction = np.zeros(y.shape)\n", "    A1, cache1 = linear_activation_forward(X, W1, b1, \"relu\")\n", "    A2, cache2 = linear_activation_forward(A1, W2, b2, \"sigmoid\")\n", "    y_prediction = A2 > 0.5\n", "    y_prediction = y_prediction.astype(float)\n", "    accuracy = 100 - np.mean(np.abs(y_prediction -y))\n", "    return accuracy"], "metadata": {"_uuid": "b9d9887e3c1ccf4ba77483be027c73667c4011e6", "collapsed": true, "_cell_guid": "dafc3e9d-9f65-40c4-b24c-9356fec8f6be"}, "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "6d39e74a7e0eb029f1efcdd12426002d15cf47ba", "collapsed": true, "_cell_guid": "9f766645-e000-4e99-a557-c452868e464b"}, "cell_type": "code", "source": ["def L_model_predict(X, Y, parameters):\n", "    n_y = np.max(Y)+1\n", "    Y = one_hot_matrix(Y, n_y)\n", "    L = len(parameters)//2\n", "    A = X\n", "    for l in range(1,L):\n", "        A_prev = A\n", "        A, cache = linear_activation_forward(A_prev, parameters[\"W\" + str(l)], parameters[\"b\" + str(l)],\"relu\")\n", "    AL, cache = linear_activation_forward(A, parameters[\"W\" + str(L)], parameters[\"b\" + str(L)], \"sigmoid\")\n", "    y_prediction = AL > 0.5\n", "    y_prediction = y_prediction.astype(float)\n", "    accuracy = 100 - np.mean(np.abs(y_prediction - Y))\n", "    return accuracy"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "399310067d7820b20ce5964db3ab131d3f359230", "_cell_guid": "98da26c4-05fa-4b23-b0a9-3691a088ca9d"}, "cell_type": "code", "source": ["train_accuracy = L_model_predict(train_x.T, train_y, parameters)\n", "print(\"train accuracy: \"+str(train_accuracy)+\"%\")"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "57e973787c807813bbdc9f0e9dd95f1d492f7df6", "collapsed": true, "_cell_guid": "919705f7-d332-4911-892b-26ba54fe78b0"}, "cell_type": "code", "source": ["def decode_one_hot_matrix(mat):\n", "    vec = np.argmax(mat, axis = 0)\n", "    return vec"]}, {"source": ["def predicted_digit(X,parameters):\n", "    W1 = parameters[\"W1\"]\n", "    b1 = parameters[\"b1\"]\n", "    W2 = parameters[\"W2\"]\n", "    b2 = parameters[\"b2\"]\n", "    y_prediction = np.zeros((b2.shape[0],X.shape[1]),np.float)\n", "    A1, cache1 = linear_activation_forward(X, W1, b1, \"relu\")\n", "    A2, cache2 = linear_activation_forward(A1, W2, b2, \"sigmoid\")\n", "    y_prediction = A2 > 0.5\n", "    y_prediction = y_prediction.astype(float)\n", "    y_prediction = decode_one_hot_matrix(y_prediction)\n", "    return y_prediction"], "metadata": {"_uuid": "3c86525a6b672183d2654681f218adf2d2e5a195", "collapsed": true, "_cell_guid": "e70301ab-c4fa-4462-8113-9e2f9797a76c"}, "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "e32ed554854a359885b8e64e41b31e5d5cb1a275", "collapsed": true, "_cell_guid": "240e73e6-8e60-4202-a568-a3d8326c7020"}, "cell_type": "code", "source": ["def L_model_predicted_digit(X, parameters):\n", "    L = len(parameters)//2\n", "    A = X\n", "    for l in range(1,L):\n", "        A_prev = A\n", "        A, cache = linear_activation_forward(A_prev, parameters[\"W\" + str(l)], parameters[\"b\" + str(l)],\"relu\")\n", "    AL, cache = linear_activation_forward(A, parameters[\"W\" + str(L)], parameters[\"b\" + str(L)], \"sigmoid\")\n", "    y_prediction = np.zeros((parameters[\"b\"+str(L)].shape[0],X.shape[1]),np.float)\n", "    y_prediction = AL > 0.5\n", "    y_prediction = y_prediction.astype(float)\n", "    y_prediction = decode_one_hot_matrix(y_prediction)\n", "    return y_prediction"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "045f5648dfdf906e1667e47178f4ee320ae000b6", "_cell_guid": "c844c468-bcfd-4c77-ad69-7d1f2f8a586f"}, "cell_type": "code", "source": ["test_y =L_model_predicted_digit(test.T,parameters)\n", "pd.DataFrame(test_y)"]}, {"execution_count": null, "outputs": [], "metadata": {"_kg_hide-output": false, "_kg_hide-input": false, "_uuid": "5c4306d6d96c82bc7cddcb30637e35ae659e885b", "_cell_guid": "caa5bb3e-100f-4c8f-bb2f-21e21d096078"}, "cell_type": "code", "source": ["ImageId = np.array(list(range(test_y.shape[0])))\n", "ImageId = ImageId + 1\n", "submission = pd.DataFrame({\n", "        \"ImageId\": ImageId,\n", "        \"Label\": test_y\n", "    })\n", "submission.to_csv(\"output3.csv\", index=False)\n", "outputFile = pd.read_csv(\"output3.csv\")\n", "outputFile.head()"]}, {"execution_count": null, "outputs": [], "metadata": {"_uuid": "958cf5b63ac6419942a1e851a08aaadadedffb0f", "_cell_guid": "098a7798-8e77-4863-a192-08fa2e0a00e2"}, "cell_type": "code", "source": ["index = 25000\n", "plt.figure(7)\n", "plt.imshow(test.iloc[index,:].reshape(28,28), cmap = \"gray\")\n", "print(\"Corresponding label = \" + str(test_y[index]))"]}]}