{"cells":[{"metadata":{},"cell_type":"markdown","source":"#### Deep Neural Network written in Python for the MNSIT handwritten dataset from scratch without using any deep learning frameworks. I have implemented Droupout technique for regularisation and Mini-batch, Adams optimisation for optimising the gradient descent and Sigmoid is used as an activation function."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport os\nimport warnings\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\n%matplotlib inline\n\nplt.rcParams[\"figure.figsize\"] = 10,7\n\nwarnings.filterwarnings('ignore') # filter all warnings\n\n# set a seed so that the results are consistent\nnp.random.seed(0)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nfor dirname, _, filenames in os.walk('/kaggle/input/digit-recognizer/'):\n    for filename in filenames:\n        print(filename)\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1. Load the Train and Test dataset"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ntrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"range_class = np.arange(10)\n\ny = np.asfarray(train.iloc[:,0])\ntrain_x = train.iloc[:,1:].values\n\ntrain_x, dev_x, train_y, dev_y = train_test_split(train_x, y, test_size=0.2, random_state=42)\n\ndev_ch_y = np.array([(range_class==label).astype(np.float) for label in dev_y])\ntrain_ch_y = np.array([(range_class==label).astype(np.float) for label in train_y])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_x = test.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Data Visualize"},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train.iloc[:,0].value_counts()\nx = range(len(y))\nplt.bar(x, y, color='rgbymc')\nplt.xticks(x, x)\nplt.ylabel('no. of images w.r.t labels')\nplt.xlabel('Lables between 0-9')\nplt.grid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating a figure to display images in rows and columns pattern (1x 10)\nfigure = plt.figure()\n\n# Manually setting the figure width and height\nfigure.set_size_inches(20.5, 8.5)\n\n# Setting up an image in each figure with a title of image label\nfor itr in range(1, 10):\n    plt.subplot(1, 10, itr)\n    label = train.loc[itr,'label']\n    pixels = train.iloc[itr,1:].values.reshape((28,28))\n    plt.title('Label is {label}'.format(label=label))\n    plt.imshow(pixels, cmap='gray')\n\n# Displaying all image present in figure\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. Normalising the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x = train_x / 255.\ntest_x  = test_x  / 255.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"shape_x = train_x.shape\nshape_y = train_y.shape\n\nshape_dev_x = dev_x.shape\nshape_dev_y = dev_y.shape\n\nm = train_y.shape[0]\n\nprint ('The shape of Train X is: %s' % str(shape_x))\nprint ('The shape of Train Y is: %s\\n' % str(shape_y))\nprint ('The shape of Dev X is: %s' % str(shape_dev_x))\nprint ('The shape of Dev Y is: %s\\n' % str(shape_dev_y))\nprint ('I have m = %d training examples! \\n' % (m))\nprint ('I have m = %d dev examples!' % (shape_dev_x[0]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. Defining the Neural Network"},{"metadata":{"trusted":true},"cell_type":"code","source":"def layer_size(X, Y):\n    \n    n_x = X.shape[1]\n    n_h = 4\n    n_y = Y.shape[1]\n    \n    return (n_x, n_h, n_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def initialise_parameter(n_x, n_h, n_y):\n    \n    np.random.seed(0)\n    \n    W1 = np.random.randn(n_h[0], n_x) * np.sqrt(1. / n_x)\n    b1 = np.zeros(shape=(n_h[0], 1))\n    \n    W2 = np.random.randn(n_h[1], n_h[0]) * np.sqrt(1. / n_h[0])\n    b2 = np.zeros(shape=(n_h[1], 1))\n    \n    W3 = np.random.randn(n_y, n_h[1]) * np.sqrt(1. / n_h[1])\n    b3 = np.zeros(shape=(n_y, 1))\n    \n    assert(W1.shape == (n_h[0], n_x))\n    assert(b1.shape == (n_h[0], 1))\n\n    assert(W2.shape == (n_h[1], n_h[0]))\n    assert(b2.shape == (n_h[1], 1))\n    \n    assert(W3.shape == (n_y, n_h[1]))\n    assert(b3.shape == (n_y, 1))\n    \n    parameters = {\"W1\": W1, \n                  \"b1\": b1, \n                  \"W2\": W2, \n                  \"b2\": b2, \n                  \"W3\": W3, \n                  \"b3\": b3\n                 }\n    \n    return parameters","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6. Initialise Adam parameters "},{"metadata":{"trusted":true},"cell_type":"code","source":"def initialise_adam(parameters):\n    \n    v = {}\n    s = {}\n    \n    v[\"dW1\"] = np.zeros((parameters[\"W1\"].shape[0],parameters[\"W1\"].shape[1]))\n    v[\"db1\"] = np.zeros((parameters[\"b1\"].shape[0],parameters[\"b1\"].shape[1]))\n    s[\"dW1\"] = np.zeros((parameters[\"W1\"].shape[0],parameters[\"W1\"].shape[1]))\n    s[\"db1\"] = np.zeros((parameters[\"b1\"].shape[0],parameters[\"b1\"].shape[1]))\n    \n    v[\"dW2\"] = np.zeros((parameters[\"W2\"].shape[0],parameters[\"W2\"].shape[1]))\n    v[\"db2\"] = np.zeros((parameters[\"b2\"].shape[0],parameters[\"b2\"].shape[1]))\n    s[\"dW2\"] = np.zeros((parameters[\"W2\"].shape[0],parameters[\"W2\"].shape[1]))\n    s[\"db2\"] = np.zeros((parameters[\"b2\"].shape[0],parameters[\"b2\"].shape[1]))\n    \n    v[\"dW3\"] = np.zeros((parameters[\"W3\"].shape[0],parameters[\"W3\"].shape[1]))\n    v[\"db3\"] = np.zeros((parameters[\"b3\"].shape[0],parameters[\"b3\"].shape[1]))\n    s[\"dW3\"] = np.zeros((parameters[\"W3\"].shape[0],parameters[\"W3\"].shape[1]))\n    s[\"db3\"] = np.zeros((parameters[\"b3\"].shape[0],parameters[\"b3\"].shape[1]))\n    \n    \n    return v, s","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 7. Activation Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sigmoid Function Defination\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n\ndef sigmoid_derivative(x):\n    return x * (1 - x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_x = np.linspace(-5, 5, 40)\nplt.plot(sigmoid(_x))\nplt.plot(sigmoid_derivative(sigmoid(_x)))\nplt.grid()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 8. Forward Propagation with Dropout"},{"metadata":{"trusted":true},"cell_type":"code","source":"def forward_propagation(X, parameters, keep_prob):\n    \n    W1 = parameters[\"W1\"]\n    b1 = parameters[\"b1\"]\n    W2 = parameters[\"W2\"]\n    b2 = parameters[\"b2\"]\n    W3 = parameters[\"W3\"]\n    b3 = parameters[\"b3\"]\n    \n    Z1 = (np.dot(W1, X.T) + b1).T\n    A1 = sigmoid(Z1)\n    \n    D1 = np.random.rand(A1.shape[0], A1.shape[1])\n    D1 = (D1 < keep_prob).astype(int)\n    A1 = A1 * D1\n    A1 = A1 / keep_prob\n    \n    Z2 = (np.dot(W2, A1.T) + b2).T\n    A2 = sigmoid(Z2)\n    \n    D2 = np.random.rand(A2.shape[0], A2.shape[1])\n    D2 = (D2 < keep_prob).astype(int)\n    A2 = A2 * D2\n    A2 = A2 / keep_prob\n    \n    Z3 = (np.dot(W3, A2.T) + b3).T\n    A3 = sigmoid(Z3)\n    \n    assert(A3.shape == (X.shape[0], 10))\n    \n    cache = {\n        \"Z1\" : Z1,\n        \"A1\" : A1,\n        \"Z2\" : Z2,\n        \"A2\" : A2,\n        \"Z3\" : Z3,\n        \"A3\" : A3,\n        \"D1\" : D1,\n        \"D2\" : D2\n    }\n\n    return A3, cache","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 9. Cost Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_cost(A3, Y):\n    \n    m = Y.shape[0] # number of example\n    \n    logprobs = np.multiply(Y, np.log(A3)) + np.multiply((1 - Y), np.log(1 - A3))\n    cost = - np.sum(logprobs) / m\n    \n    cost = float(np.squeeze(cost))\n    \n    assert(isinstance(cost, float))\n    \n    return cost","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 10. Backward Propagation with Dropout"},{"metadata":{"trusted":true},"cell_type":"code","source":"def backward_propagation(parameters, cache, X, Y, keep_prob):\n    \n    m = Y.shape[0]\n    \n    A1 = cache[\"A1\"]\n    A2 = cache[\"A2\"]\n    A3 = cache[\"A3\"]\n    \n    D1 = cache[\"D1\"]\n    D2 = cache[\"D2\"]\n    \n    W1 = parameters[\"W1\"]\n    W2 = parameters[\"W2\"]\n    W3 = parameters[\"W3\"]\n    \n    dZ3 = (A3 - Y)\n    dW3 = (1 / m) * np.dot(dZ3.T, A2)\n    db3 = (1 / m) * np.sum(dZ3, keepdims=True)\n\n    dA2 = np.dot(dZ3, W3)\n    dA2 = dA2 * D2\n    dA2 = dA2 / keep_prob\n    \n    dZ2 = np.multiply(dA2, sigmoid_derivative(A2))\n    dW2 = (1 / m) * np.dot(dZ2.T, A1)\n    db2 = (1 / m) * np.sum(dZ2, keepdims=True)\n    \n    dA1 = np.dot(dZ2, W2)\n    dA1 = dA1 * D1\n    dA1 = dA1 / keep_prob\n    \n    dZ1 = np.multiply(dA1, sigmoid_derivative(A1))\n    dW1 = (1 / m) * np.dot(dZ1.T, X)\n    db1 = (1 / m) * np.sum(dZ1, keepdims=True)\n    \n    grads = {\"dW1\": dW1, \n             \"db1\": db1, \n             \"dW2\": dW2, \n             \"db2\": db2, \n             \"dW3\": dW3, \n             \"db3\": db3\n            }\n    \n    return grads\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 11. Update Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_parameters(parameters, grads, learning_rate):\n    \n    W1 = parameters[\"W1\"]\n    b1 = parameters[\"b1\"]\n    W2 = parameters[\"W2\"]\n    b2 = parameters[\"b2\"]\n    W3 = parameters[\"W3\"]\n    b3 = parameters[\"b3\"]\n    \n    dW1 = grads[\"dW1\"]\n    db1 = grads[\"db1\"]\n    dW2 = grads[\"dW2\"]\n    db2 = grads[\"db2\"]\n    dW3 = grads[\"dW3\"]\n    db3 = grads[\"db3\"]\n    \n    W1 = W1 - (learning_rate * dW1)\n    b1 = b1 - (learning_rate * db1)\n    W2 = W2 - (learning_rate * dW2)\n    b2 = b2 - (learning_rate * db2)\n    W3 = W3 - (learning_rate * dW3)\n    b3 = b3 - (learning_rate * db3)\n    \n    parameters = {\"W1\": W1, \n                  \"b1\": b1, \n                  \"W2\": W2, \n                  \"b2\": b2, \n                  \"W3\": W3, \n                  \"b3\": b3\n                 }\n    \n    return parameters","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 12. Update parameter with Adam"},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_parameters_with_adam(parameters, grads, v, s, learning_rate, t = 2, beta1 = 0.9, beta2 = 0.999,  epsilon = 1e-8):\n    \n    v_corrected = {}\n    s_corrected = {}\n    L = len(parameters) // 2\n    \n    for l in range(L):\n\n        v[\"dW\" + str(l+1)] = (beta1 * v[\"dW\" + str(l+1)]) + ((1 - beta1) * grads[\"dW\" + str(l+1)])\n        v[\"db\" + str(l+1)] = (beta1 * v[\"db\" + str(l+1)]) + ((1 - beta1) * grads[\"db\" + str(l+1)])\n\n        v_corrected[\"dW\" + str(l+1)] = v[\"dW\" + str(l+1)] / (1 - np.power(beta1, t))\n        v_corrected[\"db\" + str(l+1)] = v[\"db\" + str(l+1)] / (1 - np.power(beta1, t))\n\n        s[\"dW\" + str(l+1)] = (beta2 * s[\"dW\" + str(l+1)]) + ((1 - beta2) * np.power(grads[\"dW\" + str(l+1)], 2))\n        s[\"db\" + str(l+1)] = (beta2 * s[\"db\" + str(l+1)]) + ((1 - beta2) * np.power(grads[\"db\" + str(l+1)], 2))\n\n        s_corrected[\"dW\" + str(l+1)] = s[\"dW\" + str(l+1)] / (1 - np.power(beta2, t))\n        s_corrected[\"db\" + str(l+1)] = s[\"db\" + str(l+1)] / (1 - np.power(beta2, t))\n\n        parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * \\\n                                        (v_corrected[\"dW\" + str(l+1)]/ np.sqrt(s_corrected[\"dW\" + str(l+1)] + epsilon))\n        \n        parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * \\\n                                        (v_corrected[\"db\" + str(l+1)]/ np.sqrt(s_corrected[\"db\" + str(l+1)] + epsilon))\n\n    return parameters, v, s","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 12. Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(X, parameters, keep_prob):\n    \n    m = X.shape[0]\n    \n    A3, cache = forward_propagation(X, parameters, keep_prob = 1.0)\n    \n    return A3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 13. Split Dataset in Batches"},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_in_mini_batches(X, Y, mini_batch_size = 128):\n    \n    m = X.shape[0]\n\n    mini_batches = []\n\n    len = int(X.shape[0]/mini_batch_size)\n\n    for k in range(0, len):\n        mini_batch_x = X[mini_batch_size * k : mini_batch_size * (k + 1), :]\n        mini_batch_y = Y[mini_batch_size * k : mini_batch_size * (k + 1), :]\n\n        assert(mini_batch_x.shape == (mini_batch_size, 784))\n        assert(mini_batch_y.shape == (mini_batch_size, 10))\n\n        mini_batch = (mini_batch_x, mini_batch_y)\n        mini_batches.append(mini_batch)\n\n    if m % mini_batch_size != 0:\n        mini_batch_x = X[mini_batch_size * (k + 1) : m, :]\n        mini_batch_y = Y[mini_batch_size * (k + 1) : m, :]\n        \n        mini_batch = (mini_batch_x, mini_batch_y)\n        mini_batches.append(mini_batch)\n        \n    return mini_batches","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 14. Neural Network Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def nn_model(X, Y, n_h, learning_rate, num_iterations, keep_prob, mini_batch_size, print_cost=False):\n\n    np.random.seed(3)\n\n    cost_per_iter = []\n    \n    dev_accuracy_arr = []\n    train_accuracy_arr = []\n    \n    n_x = layer_size(X, Y)[0]\n    n_y = layer_size(X, Y)[2]\n    \n    # Initialize parameters\n    parameters = initialise_parameter(n_x, n_h, n_y)\n    v, s = initialise_adam(parameters)\n    \n    mini_batches = split_in_mini_batches(X, Y, mini_batch_size)\n    \n    # Loop (gradient descent)\n    for i in range(0, num_iterations):\n        \n        for m_x, m_y in mini_batches:\n        \n            # Forward propagation. Inputs: \"X, parameters\". Outputs: \"A2, cache\".\n            A3, cache = forward_propagation(m_x, parameters, keep_prob)\n\n            # Cost function. Inputs: \"A2, Y, parameters\". Outputs: \"cost\".\n            cost = compute_cost(A3, m_y)\n            \n            cost_per_iter.append(cost)\n\n            # Backpropagation. Inputs: \"parameters, cache, X, Y\". Outputs: \"grads\".\n            grads = backward_propagation(parameters, cache, m_x, m_y, keep_prob)\n\n            # Gradient descent parameter update. Inputs: \"parameters, grads\". Outputs: \"parameters\".\n            parameters, v, s = update_parameters_with_adam(parameters, grads, v, s, learning_rate)\n            \n            # Print the cost every 1000 iterations\n            if print_cost and i % 100 == 0:\n                print (\"Cost after iteration %i: %f\" %(i, cost))\n\n            train_prediction = (train_y != np.array(predict(train_x, parameters, keep_prob).argmax(axis=1)).T).astype(int)\n            dev_prediction = (dev_y != np.array(predict(dev_x, parameters, keep_prob).argmax(axis=1)).T).astype(int)\n\n            dev_accuracy_arr.append(100 - np.mean(dev_prediction) * 100)\n            train_accuracy_arr.append(100 - np.mean(train_prediction) * 100)\n    \n    test_prediction = np.vstack((np.arange(1,28001), predict(test_x, parameters, keep_prob = 1.0).argmax(axis=1).T)).T\n    data_to_submit = pd.DataFrame(test_prediction, columns = ['ImageId','Label']) \n    \n    output = {\n        \"cost\" : cost_per_iter[-1],\n        \"parameters\" : parameters,\n        \"cost_per_iter\" : cost_per_iter,\n        \"train_accuracy_arr\" : train_accuracy_arr,\n        \"dev_accuracy_arr\" : dev_accuracy_arr,\n        \"data_to_submit\" : data_to_submit\n    }\n    \n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = {}\nlearning_rates = [0.001]\n\nfor i in learning_rates:\n    print (\"learning rate is: \" + str(i))\n    \n    models[str(i)] = nn_model(train_x, train_ch_y, n_h = [500, 50], learning_rate = i, \\\n                              num_iterations = 20, keep_prob = 0.8, mini_batch_size = 64, print_cost=False)\n    \n    print (\"Cost is: \" + str(models[str(i)][\"cost\"]))\n    print(\"train accuracy: {} %\".format(models[str(i)][\"train_accuracy_arr\"][-1]))\n    print(\"dev accuracy: {} %\".format(models[str(i)][\"dev_accuracy_arr\"][-1]))\n    print (\"-------------------------------------------------------\" + '\\n')\n\nfor i in learning_rates:\n    plt.plot(np.squeeze(models[str(i)][\"cost_per_iter\"]), label= str(i))\n\nplt.ylabel('cost')\nplt.xlabel('iterations')\nplt.legend(loc='upper right')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(np.squeeze(models[\"0.001\"][\"dev_accuracy_arr\"]), label= \"Dev Accuracy\")\nplt.plot(np.squeeze(models[\"0.001\"][\"train_accuracy_arr\"]), label= \"Train Accuracy\")\nplt.legend(loc='upper left')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models[\"0.001\"][\"data_to_submit\"].to_csv('csv_to_submit.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}