{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Here, I attempt to build a simple feed forward neural network with one hidden layer, to identify and classify handwritten digits","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt # visualization\nimport sklearn # a possible requirement for dividing training and test sets\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-18T08:46:49.063930Z","iopub.execute_input":"2022-12-18T08:46:49.064323Z","iopub.status.idle":"2022-12-18T08:46:49.071496Z","shell.execute_reply.started":"2022-12-18T08:46:49.064291Z","shell.execute_reply":"2022-12-18T08:46:49.069559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:46:51.043715Z","iopub.execute_input":"2022-12-18T08:46:51.044112Z","iopub.status.idle":"2022-12-18T08:46:54.981452Z","shell.execute_reply.started":"2022-12-18T08:46:51.044081Z","shell.execute_reply":"2022-12-18T08:46:54.980282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:46:54.983542Z","iopub.execute_input":"2022-12-18T08:46:54.984288Z","iopub.status.idle":"2022-12-18T08:46:54.989673Z","shell.execute_reply.started":"2022-12-18T08:46:54.984248Z","shell.execute_reply":"2022-12-18T08:46:54.987987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.columns","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:46:59.602087Z","iopub.execute_input":"2022-12-18T08:46:59.603153Z","iopub.status.idle":"2022-12-18T08:46:59.609719Z","shell.execute_reply.started":"2022-12-18T08:46:59.603118Z","shell.execute_reply":"2022-12-18T08:46:59.608608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.columns","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:47:07.066859Z","iopub.execute_input":"2022-12-18T08:47:07.067263Z","iopub.status.idle":"2022-12-18T08:47:07.075055Z","shell.execute_reply.started":"2022-12-18T08:47:07.067228Z","shell.execute_reply":"2022-12-18T08:47:07.073982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The DataFrame 'data' has 784 pixel values from columns indexed 1 - 784.\n\nThe first column the label i.e. the digit that each set of pixels is meant to be.","metadata":{}},{"cell_type":"code","source":"data = np.array(data)\ntest = np.array(test)\nm, n = data.shape\nnp.random.shuffle(data)\n\n# I wish to divide the training and development sets in the ratio of 80:20\n# This could have also been done with\n# X_train, X_dev, Y_train, Y_dev = train_test_split(data[1:n], data[0], test_size=0.2)\n# This is to be done before converting the dataframes into numpy arrays\n\ndata_dev = data[0:8400].T\nY_dev = data_dev[0]\nX_dev = data_dev[1:n]/255 # For faster convergence\n\ndata_train = data[8400:m].T\nY_train = data_train[0]\nX_train = data_train[1:n]/255\n_, m_train = X_train.shape\n\nX_test = test.T\nX_test = X_test/255","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:47:10.012229Z","iopub.execute_input":"2022-12-18T08:47:10.013642Z","iopub.status.idle":"2022-12-18T08:47:10.982695Z","shell.execute_reply.started":"2022-12-18T08:47:10.013598Z","shell.execute_reply":"2022-12-18T08:47:10.980829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_params(size, hidden_layer_nodes):\n    '''Initializing the weight and bias arrays for the \n    hidden and output layers'''\n    W1 = np.random.rand(hidden_layer_nodes, size) - 0.5 # Values range from -0.5 to 0.5\n    b1 = np.random.rand(hidden_layer_nodes, 1) - 0.5\n    W2 = np.random.rand(10, hidden_layer_nodes) - 0.5\n    b2 = np.random.rand(10, 1) - 0.5\n    return W1, b1, W2, b2\n\ndef ReLU(Z):\n    return np.maximum(0, Z) # An elegant way of saying: return X if X > 0 and 0 otherwise.\n\ndef de_ReLU(Z):\n    return Z > 0 # The derivative of Z when Z is greater than 0 is 1.\n\ndef softmax(Z):\n    return np.exp(Z)/np.sum(np.exp(Z), axis = 0)\n\ndef forward_prop(W1, b1, W2, b2, X):\n    '''Returns the outputs of linear units and the \n    activations of both layers of neurons'''\n    Z1 = W1.dot(X) + b1\n    A1 = ReLU(Z1)\n    Z2 = W2.dot(A1) + b2\n    A2 = softmax(Z2)\n    return Z1, A1, Z2, A2\n\ndef one_hot(Y):\n    '''Encoding required for classification problems'''\n    one_hot_Y = np.zeros((Y.max() + 1, Y.size))\n    one_hot_Y[Y, np.arange(Y.size)] = 1\n    return one_hot_Y\n\ndef back_prop(Z1, A1, Z2, A2, W2, X, Y):\n    m = Y.size\n    one_hot_Y = one_hot(Y)\n    dZ2 = 2*(A2 - one_hot_Y)\n    dW2 = 1/m * (dZ2.dot(A1.T))\n    db2 = 1/m * np.sum(dZ2, 1)\n    dZ1 = W2.T.dot(dZ2) * de_ReLU(Z1)\n    dW1 = 1/m * (dZ1.dot(X.T))\n    db1 = 1/m * np.sum(dZ1, 1)\n    return dW1, db1, dW2, db2\n\ndef update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, rate, hidden_layer_nodes):\n    W1 -= rate * dW1\n    b1 -= rate * np.reshape(db1, (hidden_layer_nodes, 1))\n    W2 -= rate * dW2\n    b2 -= rate * np.reshape(db2, (10, 1))\n    return W1, b1, W2, b2","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:47:11.031359Z","iopub.execute_input":"2022-12-18T08:47:11.032102Z","iopub.status.idle":"2022-12-18T08:47:11.056833Z","shell.execute_reply.started":"2022-12-18T08:47:11.032051Z","shell.execute_reply":"2022-12-18T08:47:11.055589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(A2):\n    return np.argmax(A2, 0)\n\ndef get_accuracy(predictions, Y):\n    print(predictions, Y)\n    return np.sum(predictions == Y)/Y.size\n\ndef gradient_descent(X, Y, iterations, rate, hidden_layer_nodes):\n    size, m = X.shape\n    W1, b1, W2, b2 = init_params(size, hidden_layer_nodes)\n    acc = 0\n    for i in range(iterations):\n        Z1, A1, Z2, A2 = forward_prop(W1, b1, W2, b2, X)\n        dW1, db1, dW2, db2 = back_prop(Z1, A1, Z2, A2, W2, X, Y)\n        W1, b1, W2, b2 = update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, rate, hidden_layer_nodes)\n        if (i + 1)% (iterations/10) == 0:\n            acc = get_accuracy(get_predictions(A2), Y)\n            print('iteration', i + 1)\n            print('accuracy:', acc)\n    return W1, b1, W2, b2, acc","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:47:11.609704Z","iopub.execute_input":"2022-12-18T08:47:11.610111Z","iopub.status.idle":"2022-12-18T08:47:11.620550Z","shell.execute_reply.started":"2022-12-18T08:47:11.610077Z","shell.execute_reply":"2022-12-18T08:47:11.619161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Time to train our network!","metadata":{}},{"cell_type":"code","source":"W1, b1, W2, b2, final_accuracy = gradient_descent(X_train, Y_train, 500, 0.1, 10)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:47:13.506371Z","iopub.execute_input":"2022-12-18T08:47:13.506830Z","iopub.status.idle":"2022-12-18T08:48:17.212122Z","shell.execute_reply.started":"2022-12-18T08:47:13.506783Z","shell.execute_reply":"2022-12-18T08:48:17.210494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"An accuracy of over 88%!","metadata":{}},{"cell_type":"code","source":"def make_prediction(W1, b1, W2, b2, X):\n    _, _, _, A2 = forward_prop(W1, b1, W2, b2, X)\n    pred = get_predictions(A2)\n    return pred\n\ndef test_prediction(index, W1, b1, W2, b2, X):\n    X = X[:, index, None]\n    pred = make_prediction(W1, b1, W2, b2, X)\n    print(\"Prediction:\", pred)\n    current_image = X.reshape((28, 28)) * 255\n    plt.gray()\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:48:33.282272Z","iopub.execute_input":"2022-12-18T08:48:33.282722Z","iopub.status.idle":"2022-12-18T08:48:33.291551Z","shell.execute_reply.started":"2022-12-18T08:48:33.282686Z","shell.execute_reply":"2022-12-18T08:48:33.290281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_prediction(0, W1, b1, W2, b2, X_train)\ntest_prediction(1, W1, b1, W2, b2, X_train)\ntest_prediction(2, W1, b1, W2, b2, X_train)\ntest_prediction(3, W1, b1, W2, b2, X_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:48:34.138780Z","iopub.execute_input":"2022-12-18T08:48:34.139192Z","iopub.status.idle":"2022-12-18T08:48:35.232604Z","shell.execute_reply.started":"2022-12-18T08:48:34.139159Z","shell.execute_reply":"2022-12-18T08:48:35.231209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's take a look at our development set.","metadata":{}},{"cell_type":"code","source":"dev_pred = make_prediction(W1, b1, W2, b2, X_dev)\nget_accuracy(dev_pred, Y_dev)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:48:38.686245Z","iopub.execute_input":"2022-12-18T08:48:38.686670Z","iopub.status.idle":"2022-12-18T08:48:38.747337Z","shell.execute_reply.started":"2022-12-18T08:48:38.686635Z","shell.execute_reply":"2022-12-18T08:48:38.745610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Over 88% accuracy here too! So there's no real need to train our network according to the dev set.\n\nNow, let's test it using unlabeled data. This is of great importance, since in real-world scenarios, the test set is never available to you during training.","metadata":{}},{"cell_type":"code","source":"def final_test(index, W1, b1, W2, b2):\n    global X_test\n    test_prediction(index, W1, b1, W2, b2, X_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:49:13.550948Z","iopub.execute_input":"2022-12-18T08:49:13.551344Z","iopub.status.idle":"2022-12-18T08:49:13.558215Z","shell.execute_reply.started":"2022-12-18T08:49:13.551311Z","shell.execute_reply":"2022-12-18T08:49:13.556102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(20):\n    final_test(i, W1, b1, W2, b2)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T08:49:17.335933Z","iopub.execute_input":"2022-12-18T08:49:17.336386Z","iopub.status.idle":"2022-12-18T08:49:21.320239Z","shell.execute_reply.started":"2022-12-18T08:49:17.336348Z","shell.execute_reply":"2022-12-18T08:49:21.318396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We're doing pretty alright!\n\nThere are some inaccuracies apparent, like indexes 2, 3 and 4. Apart from those however, we got everything correct.","metadata":{}},{"cell_type":"markdown","source":"An excellent video series on neural networks is available [here](https://www.youtube.com/watch?v=aircAruvnKk&list=PLZHQObOWTQDNU6R1_67000Dx_ZCJB-3pi).\n\nInspiration and instructions for this project come from [here](https://youtu.be/w8yWXqWQYmU).\n\nI would one day like to build a shape-recognizing neural network, rather than a network that studyies the activations of pixels in a restricted 28x28 space.\n\n","metadata":{}}]}