{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Image Classifier using MNIST \n\nIn this notebook, I implemented a simple neural network with one hidden layer, Activation and Sigmoid functions and trained it on the MNIST dataset. This features both Forward and Reverse Propogation to calibrate all Weights and Biases accordingly.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\ndata = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-06T22:03:08.563942Z","iopub.execute_input":"2022-05-06T22:03:08.564298Z","iopub.status.idle":"2022-05-06T22:03:12.535488Z","shell.execute_reply.started":"2022-05-06T22:03:08.564268Z","shell.execute_reply":"2022-05-06T22:03:12.534227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = np.array(data)\nm, n = data.shape\nnp.random.shuffle(data) \n\ndata_dev = data[0:1000].T\nY_dev = data_dev[0]\nX_dev = data_dev[1:n]\nX_dev = X_dev / 255.\n\ndata_train = data[1000:m].T\nY_train = data_train[0]\nX_train = data_train[1:n]\nX_train = X_train / 255.\n_,m_train = X_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:03:12.538915Z","iopub.execute_input":"2022-05-06T22:03:12.539275Z","iopub.status.idle":"2022-05-06T22:03:13.042871Z","shell.execute_reply.started":"2022-05-06T22:03:12.539245Z","shell.execute_reply":"2022-05-06T22:03:13.041253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train ","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:03:13.044492Z","iopub.execute_input":"2022-05-06T22:03:13.044827Z","iopub.status.idle":"2022-05-06T22:03:13.054498Z","shell.execute_reply.started":"2022-05-06T22:03:13.044797Z","shell.execute_reply":"2022-05-06T22:03:13.053143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Our Neural Network will have a simple two-layer architecture. Input layer $a^{[0]}$ will have 784 units corresponding to the 784 pixels in each 28x28 input image. A hidden layer $a^{[1]}$ will have 10 units with ReLU activation, and finally our output layer $a^{[2]}$ will have 10 units corresponding to the ten digit classes with normalized exponential function.\n\n**Forward propagation**\n\n$$Z^{[1]} = W^{[1]} X + b^{[1]}$$\n$$A^{[1]} = g_{\\text{ReLU}}(Z^{[1]}))$$\n$$Z^{[2]} = W^{[2]} A^{[1]} + b^{[2]}$$\n$$A^{[2]} = g_{\\text{softmax}}(Z^{[2]})$$\n\n**Backward propagation**\n\n$$dZ^{[2]} = A^{[2]} - Y$$\n$$dW^{[2]} = \\frac{1}{m} dZ^{[2]} A^{[1]T}$$\n$$dB^{[2]} = \\frac{1}{m} \\Sigma {dZ^{[2]}}$$\n$$dZ^{[1]} = W^{[2]T} dZ^{[2]} .* g^{[1]\\prime} (z^{[1]})$$\n$$dW^{[1]} = \\frac{1}{m} dZ^{[1]} A^{[0]T}$$\n$$dB^{[1]} = \\frac{1}{m} \\Sigma {dZ^{[1]}}$$\n\n**Parameter updates**\n\n$$W^{[2]} := W^{[2]} - \\alpha dW^{[2]}$$\n$$b^{[2]} := b^{[2]} - \\alpha db^{[2]}$$\n$$W^{[1]} := W^{[1]} - \\alpha dW^{[1]}$$\n$$b^{[1]} := b^{[1]} - \\alpha db^{[1]}$$\n\n**Vars and shapes**\n\nForward prop\n\n- $A^{[0]} = X$: 784 x m\n- $Z^{[1]} \\sim A^{[1]}$: 10 x m\n- $W^{[1]}$: 10 x 784 (as $W^{[1]} A^{[0]} \\sim Z^{[1]}$)\n- $B^{[1]}$: 10 x 1\n- $Z^{[2]} \\sim A^{[2]}$: 10 x m\n- $W^{[1]}$: 10 x 10 (as $W^{[2]} A^{[1]} \\sim Z^{[2]}$)\n- $B^{[2]}$: 10 x 1\n\nBackprop\n\n- $dZ^{[2]}$: 10 x m ($~A^{[2]}$)\n- $dW^{[2]}$: 10 x 10\n- $dB^{[2]}$: 10 x 1\n- $dZ^{[1]}$: 10 x m ($~A^{[1]}$)\n- $dW^{[1]}$: 10 x 10\n- $dB^{[1]}$: 10 x 1","metadata":{}},{"cell_type":"code","source":"def init_params():\n    W1 = np.random.rand(10, 784) - 0.5\n    b1 = np.random.rand(10, 1) - 0.5\n    W2 = np.random.rand(10, 10) - 0.5\n    b2 = np.random.rand(10, 1) - 0.5\n    return W1, b1, W2, b2\n\ndef ReLU(Z):\n    return np.maximum(Z, 0)\n\ndef softmax(Z):\n    A = np.exp(Z) / sum(np.exp(Z))\n    return A\n    \ndef forward_prop(W1, b1, W2, b2, X):\n    Z1 = W1.dot(X) + b1\n    A1 = ReLU(Z1)\n    Z2 = W2.dot(A1) + b2\n    A2 = softmax(Z2)\n    return Z1, A1, Z2, A2\n\ndef ReLU_deriv(Z):\n    return Z > 0\n\ndef one_hot(Y):\n    one_hot_Y = np.zeros((Y.size, Y.max() + 1))\n    one_hot_Y[np.arange(Y.size), Y] = 1\n    one_hot_Y = one_hot_Y.T\n    return one_hot_Y\n\ndef backward_prop(Z1, A1, Z2, A2, W1, W2, X, Y):\n    one_hot_Y = one_hot(Y)\n    dZ2 = A2 - one_hot_Y\n    dW2 = 1 / m * dZ2.dot(A1.T)\n    db2 = 1 / m * np.sum(dZ2)\n    dZ1 = W2.T.dot(dZ2) * ReLU_deriv(Z1)\n    dW1 = 1 / m * dZ1.dot(X.T)\n    db1 = 1 / m * np.sum(dZ1)\n    return dW1, db1, dW2, db2\n\ndef update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, alpha):\n    W1 = W1 - alpha * dW1\n    b1 = b1 - alpha * db1    \n    W2 = W2 - alpha * dW2  \n    b2 = b2 - alpha * db2    \n    return W1, b1, W2, b2","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:03:13.055876Z","iopub.execute_input":"2022-05-06T22:03:13.056210Z","iopub.status.idle":"2022-05-06T22:03:13.078753Z","shell.execute_reply.started":"2022-05-06T22:03:13.056166Z","shell.execute_reply":"2022-05-06T22:03:13.078036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(A2):\n    return np.argmax(A2, 0)\n\ndef get_accuracy(predictions, Y):\n    print(predictions, Y)\n    return np.sum(predictions == Y) / Y.size\n\ndef gradient_descent(X, Y, alpha, iterations):\n    W1, b1, W2, b2 = init_params()\n    for i in range(iterations):\n        Z1, A1, Z2, A2 = forward_prop(W1, b1, W2, b2, X)\n        dW1, db1, dW2, db2 = backward_prop(Z1, A1, Z2, A2, W1, W2, X, Y)\n        W1, b1, W2, b2 = update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, alpha)\n        if i % 10 == 0:\n            print(\"Iteration: \", i)\n            predictions = get_predictions(A2)\n            print(get_accuracy(predictions, Y))\n    return W1, b1, W2, b2","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:03:13.080621Z","iopub.execute_input":"2022-05-06T22:03:13.081013Z","iopub.status.idle":"2022-05-06T22:03:13.093091Z","shell.execute_reply.started":"2022-05-06T22:03:13.080975Z","shell.execute_reply":"2022-05-06T22:03:13.092008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"W1, b1, W2, b2 = gradient_descent(X_train, Y_train, 0.10, 500)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:03:13.094586Z","iopub.execute_input":"2022-05-06T22:03:13.094946Z","iopub.status.idle":"2022-05-06T22:04:12.658536Z","shell.execute_reply.started":"2022-05-06T22:03:13.094910Z","shell.execute_reply":"2022-05-06T22:04:12.657451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"~85% accuracy on training set.","metadata":{}},{"cell_type":"code","source":"def make_predictions(X, W1, b1, W2, b2):\n    _, _, _, A2 = forward_prop(W1, b1, W2, b2, X)\n    predictions = get_predictions(A2)\n    return predictions\n\ndef test_prediction(index, W1, b1, W2, b2):\n    current_image = X_train[:, index, None]\n    prediction = make_predictions(X_train[:, index, None], W1, b1, W2, b2)\n    label = Y_train[index]\n    print(\"Prediction: \", prediction)\n    print(\"Label: \", label)\n    \n    current_image = current_image.reshape((28, 28)) * 255\n    plt.gray()\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:04:12.659883Z","iopub.execute_input":"2022-05-06T22:04:12.660177Z","iopub.status.idle":"2022-05-06T22:04:12.670003Z","shell.execute_reply.started":"2022-05-06T22:04:12.660148Z","shell.execute_reply":"2022-05-06T22:04:12.668864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_prediction(0, W1, b1, W2, b2)\ntest_prediction(4800, W1, b1, W2, b2)\ntest_prediction(40900, W1, b1, W2, b2)\ntest_prediction(9999, W1, b1, W2, b2)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:41:10.133410Z","iopub.execute_input":"2022-05-06T22:41:10.133741Z","iopub.status.idle":"2022-05-06T22:41:10.691254Z","shell.execute_reply.started":"2022-05-06T22:41:10.133702Z","shell.execute_reply":"2022-05-06T22:41:10.690397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, let's check the accuracy","metadata":{}},{"cell_type":"code","source":"dev_predictions = make_predictions(X_dev, W1, b1, W2, b2)\nget_accuracy(dev_predictions, Y_dev)","metadata":{"execution":{"iopub.status.busy":"2022-05-06T22:04:13.354346Z","iopub.execute_input":"2022-05-06T22:04:13.354649Z","iopub.status.idle":"2022-05-06T22:04:13.373499Z","shell.execute_reply.started":"2022-05-06T22:04:13.354619Z","shell.execute_reply":"2022-05-06T22:04:13.372325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Our Neural Network is around 85% accurate, which is quite good for one with a single Hidden layer.","metadata":{}}]}