{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Advance Version of Neural Network -- MNIST\n### Goal: Predict the number written in the 28x28 pixel image","metadata":{}},{"cell_type":"markdown","source":"### The model has the following specs:\n- The first layer is the input layer, with size 784\n- Then, there are three hidden layers,\n  - The first hidden layer has 256 neurons, using Swish activation function\n  - The second hidden layer has 32 neurons, using ReLU activation function\n  - The third hidden layer has 10 neurons, using Softmax activation function\n- Then, here comes the output.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\ndata = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ndata = np.array(data)","metadata":{"execution":{"iopub.status.busy":"2024-09-12T17:59:56.978147Z","iopub.execute_input":"2024-09-12T17:59:56.978606Z","iopub.status.idle":"2024-09-12T18:00:02.099615Z","shell.execute_reply.started":"2024-09-12T17:59:56.978553Z","shell.execute_reply":"2024-09-12T18:00:02.098405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Manipulate and normalize the data","metadata":{}},{"cell_type":"markdown","source":"### Here, we split the data into two sets:\n  - The first set is for training purposes, where we train the model using the above neural network\n  - The second set is for development purposes, where we test the accuracy of the model","metadata":{}},{"cell_type":"code","source":"m, n = data.shape\nprint(f\"The data is {m} x {n}\")\nnp.random.shuffle(data)\n\ndata_dev_length = 10000\n\n# dev sev\ndata_dev = data[0:data_dev_length]\nY_dev =  data_dev[:, [0]] # (m x 1) matrix\nX_dev = data_dev[:, 1:] # (m x 784) matrix\nX_dev = X_dev / 255. # normalize the X\n\n# train set\ndata_train = data[data_dev_length:]\nY_train = data_train[:, [0]] # (m x 1) matrix\nX_train = data_train[:, 1:] # (m x 784) matrix\nX_train = X_train / 255.","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:02.102482Z","iopub.execute_input":"2024-09-12T18:00:02.103006Z","iopub.status.idle":"2024-09-12T18:00:02.835898Z","shell.execute_reply.started":"2024-09-12T18:00:02.102949Z","shell.execute_reply":"2024-09-12T18:00:02.834631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Init params and functions","metadata":{}},{"cell_type":"code","source":"# X is (m x 784)\n# Y is (m x 1)\ndef init_params():\n  def xavier_init(n_in, n_out):\n    limit = np.sqrt(6 / (n_in + n_out))\n    return np.random.uniform(-limit, limit, (n_in, n_out))\n  \n  def he_init(n_in, n_out):\n    return np.random.randn(n_in, n_out) * np.sqrt(2. / n_in)\n\n  W1 = xavier_init(784, 256)\n  b1 = np.zeros((1, 256))\n  W2 = he_init(256, 32) # work better with ReLU\n  b2 = np.zeros((1, 32))\n  W3 = xavier_init(32, 10)\n  b3 = np.zeros((1, 10))\n  return W1, b1, W2, b2, W3, b3\n\ndef Sigmoid(Z):\n  return 1 / (1 + np.exp(-Z))\n\ndef Swish(Z):\n  return Z * Sigmoid(Z)\n\ndef Swish_derivative(Z):\n  sigmoid_Z = Sigmoid(Z)\n  return sigmoid_Z + Z * sigmoid_Z * (1 - sigmoid_Z)\n\ndef ReLU(Z):\n  return np.maximum(0, Z)\n\ndef ReLU_derivative(Z):\n  return Z > 0\n\ndef Softmax(Z):\n  max_Z = np.max(Z, axis=1, keepdims=True) # find the maximum element in each row, yield a (m x 1) matrix\n  shift_Z = Z - max_Z\n  exp_Z = np.exp(shift_Z)\n  sum_Z = np.sum(exp_Z, axis=1, keepdims=True) # sum over the elements in each row, yield a (m x 1) matrix\n  return exp_Z / sum_Z\n\ndef one_hot(Y):\n  m, _ = Y.shape\n  array_Y = Y.T[0] # convert Y to an array of size m\n  one_hot = np.zeros((m, 10))\n  one_hot[np.arange(m), array_Y] = 1\n  return one_hot\n\ndef get_prediction(A):\n  return np.argmax(A, axis=1)\n\ndef get_accuracy(prediction, Y):\n  m, _ = Y.shape\n  array_Y = Y.T[0]\n  one_hot = np.zeros((m, 10))\n  return np.sum(prediction == array_Y) / m","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:02.837461Z","iopub.execute_input":"2024-09-12T18:00:02.837890Z","iopub.status.idle":"2024-09-12T18:00:02.854922Z","shell.execute_reply.started":"2024-09-12T18:00:02.837848Z","shell.execute_reply":"2024-09-12T18:00:02.853399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Neural Network -- Forward Propagation -- Backward Propagation","metadata":{}},{"cell_type":"markdown","source":"### Here, we put all the parameters used in the model in a class (OOP):\n  - W1 := 784 x 256\n  - b1 := 1 x 256\n  - W2 := 256 x 32\n  - b2 := 1 x 32\n  - W3 := 32 x 10\n  - b3 := 1 x 10","metadata":{}},{"cell_type":"code","source":"class Advance_Neural_Network:\n  def __init__(self, alpha, decay_rate, iterations, batch_size):\n    self.alpha = alpha\n    self.decay_rate = decay_rate\n    self.iterations = iterations\n    self.batch_size = batch_size\n    self.W1, self.b1, self.W2, self.b2, self.W3, self.b3 = init_params()\n\n  def forward_prop(self, X):\n    # layer 1\n    Z1 = np.dot(X, self.W1) + self.b1\n    A1 = ReLU(Z1)\n    # layer 2\n    Z2 = np.dot(A1, self.W2) + self.b2\n    A2 = ReLU(Z2)\n    # layer 3\n    Z3 = np.dot(A2, self.W3) + self.b3\n    A3 = Softmax(Z3)\n    return Z1, A1, Z2, A2, Z3, A3\n\n  def backward_prop(self, Z1, A1, Z2, A2, Z3, A3, X, Y):\n    m, _ = Y.shape\n    # layer 3\n    one_hot_Y = one_hot(Y)\n    dZ3 = (A3 - one_hot_Y) / m\n    dW3 = np.dot(A2.T, dZ3)\n    db3 = np.sum(dZ3, axis=0, keepdims=True)\n    # layer 2\n    dZ2 = np.dot(dZ3, self.W3.T) * ReLU_derivative(Z2)\n    dW2 = np.dot(A1.T, dZ2)\n    db2 = np.sum(dZ2, axis=0, keepdims=True)\n    # layer 1\n    dZ1 = np.dot(dZ2, self.W2.T) * ReLU_derivative(Z1)\n    dW1 = np.dot(X.T, dZ1)\n    db1 = np.sum(dZ1, axis=0, keepdims=True)\n    return dW1, db1, dW2, db2, dW3, db3\n\n  def update_params(self, dW1, db1, dW2, db2, dW3, db3):\n    self.W1 -= self.alpha * dW1\n    self.b1 -= self.alpha * db1\n    self.W2 -= self.alpha * dW2\n    self.b2 -= self.alpha * db2\n    self.W3 -= self.alpha * dW3\n    self.b3 -= self.alpha * db3\n  \n  def mini_batch_gradient_descent(self, X, Y):\n    # we will use a technic called mini-batch gradient descent\n    # we will randomly pick a batch of 64\n    number_of_samples, _ = X.shape\n    last_accuracy = 1e15 # init to a small number\n    for i in range(self.iterations):\n      # get a random permutation\n      permutation = np.random.permutation(number_of_samples)\n      X_shuffle = X[permutation] # shuffle X\n      Y_shuffle = Y[permutation] # shuffle Y\n      # loop\n      for j in range(0, number_of_samples, self.batch_size):\n        # get the batch_size batch\n        X_batch = X_shuffle[j:j + self.batch_size]\n        Y_batch = Y_shuffle[j:j + self.batch_size]\n        # perform ml\n        Z1, A1, Z2, A2, Z3, A3 = self.forward_prop(X_batch)\n        dW1, db1, dW2, db2, dW3, db3 = self.backward_prop(Z1, A1, Z2, A2, Z3, A3, X_batch, Y_batch)\n        self.update_params(dW1, db1, dW2, db2, dW3, db3)\n      # print the result every 50th iterations\n      if (i % 5 == 0):\n        _, _, _, _, _, A = self.forward_prop(X)\n        print(f\"Iteration: {i}\")\n        prediction = get_prediction(A)\n        accuracy = get_accuracy(prediction, Y)\n        print(f\"Accuracy: {accuracy}\")\n        # stop when the trained model is good enough\n        # prevent overfitting (the model just remmember instead of learning)\n        if ((accuracy == 1) or (accuracy >= last_accuracy and (accuracy - last_accuracy) / last_accuracy < 0.01)):\n          print(\"Stop!!!\")\n          break\n        last_accuracy = accuracy\n      # update the learning rate\n      self.alpha = self.alpha * np.exp(-self.decay_rate)","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:02.857745Z","iopub.execute_input":"2024-09-12T18:00:02.858277Z","iopub.status.idle":"2024-09-12T18:00:02.884388Z","shell.execute_reply.started":"2024-09-12T18:00:02.858236Z","shell.execute_reply":"2024-09-12T18:00:02.883104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### When training the model, it typically requires many iterations to achieve a high accuracy percentage.\n### However, by using mini-batch gradient descent, we are able to reach this level of accuracy with fewer iterations (epochs). We stop the training when the model achieves 100% accuracy or when there is no significant improvement in accuracy (less than 1%).","metadata":{}},{"cell_type":"markdown","source":"## 4. Train","metadata":{}},{"cell_type":"code","source":"alpha = 0.1\ndecay_rate = 0.01\niterations = 51\nbatch_size = 64\nmy_neural_network = Advance_Neural_Network(alpha, decay_rate, iterations, batch_size)\nmy_neural_network.mini_batch_gradient_descent(X_train, Y_train)","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:02.885971Z","iopub.execute_input":"2024-09-12T18:00:02.886352Z","iopub.status.idle":"2024-09-12T18:00:37.999756Z","shell.execute_reply.started":"2024-09-12T18:00:02.886314Z","shell.execute_reply":"2024-09-12T18:00:37.998357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The model is able to achieve very close to 100% accuracy on the train dataset.\n### Note that by using mini-batch gradient descent, the training process is more optimized than using normal gradient descent as it reduces the number of epoch and hence reduce the runtime to reach convergence by 30%.","metadata":{}},{"cell_type":"markdown","source":"## 5. Test\n### Here, we test the accuracy of the model using the development set.","metadata":{}},{"cell_type":"code","source":"def make_prediction(neural_network, X):\n  _, _, _, _, _, A3 = neural_network.forward_prop(X)\n  prediction = get_prediction(A3)\n  return prediction\n\ndef test(neural_network, X, Y):\n  prediction = make_prediction(neural_network, X)\n  print(f\"Accuracy: {get_accuracy(prediction, Y)}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:38.001430Z","iopub.execute_input":"2024-09-12T18:00:38.009836Z","iopub.status.idle":"2024-09-12T18:00:38.023182Z","shell.execute_reply.started":"2024-09-12T18:00:38.009737Z","shell.execute_reply":"2024-09-12T18:00:38.021523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test(my_neural_network, X_dev, Y_dev)","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:38.031016Z","iopub.execute_input":"2024-09-12T18:00:38.032131Z","iopub.status.idle":"2024-09-12T18:00:38.166356Z","shell.execute_reply.started":"2024-09-12T18:00:38.032052Z","shell.execute_reply":"2024-09-12T18:00:38.164790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compared to the train dataset, we see that the model has a lower accuracy but is good enough. Overall, we can say that we succesfully train this neural network model.","metadata":{}},{"cell_type":"markdown","source":"## 6. Plot some examples\n### Here, we plot some of the input images and see what number does the trained model predict.","metadata":{}},{"cell_type":"code","source":"test_data = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\ntest_data = np.array(test_data)\n\nX_test = test_data\nX_test = X_test / 255.","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:38.174181Z","iopub.execute_input":"2024-09-12T18:00:38.178444Z","iopub.status.idle":"2024-09-12T18:00:40.818257Z","shell.execute_reply.started":"2024-09-12T18:00:38.178344Z","shell.execute_reply":"2024-09-12T18:00:40.816645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_plot(neural_network, index):\n  current_image = X_test[index:index+1]\n  prediction = make_prediction(neural_network, X_test[index:index+1])\n  print(\"Prediction: \", prediction)\n  \n  current_image = current_image.reshape((28, 28)) * 255\n  plt.gray()\n  plt.imshow(current_image, interpolation='nearest')\n  plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:40.820289Z","iopub.execute_input":"2024-09-12T18:00:40.820797Z","iopub.status.idle":"2024-09-12T18:00:40.828895Z","shell.execute_reply.started":"2024-09-12T18:00:40.820740Z","shell.execute_reply":"2024-09-12T18:00:40.827427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"number_of_test = 10\nfor _ in range(number_of_test):\n  random_index = np.random.randint(0, test_data.shape[0])\n  test_plot(my_neural_network, random_index)","metadata":{"execution":{"iopub.status.busy":"2024-09-12T18:00:40.831935Z","iopub.execute_input":"2024-09-12T18:00:40.832356Z","iopub.status.idle":"2024-09-12T18:00:43.510707Z","shell.execute_reply.started":"2024-09-12T18:00:40.832314Z","shell.execute_reply":"2024-09-12T18:00:43.509579Z"},"trusted":true},"execution_count":null,"outputs":[]}]}