{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"},{"sourceId":242592,"sourceType":"datasetVersion","datasetId":102285}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\n\ndataset = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n#reads data into panadas dataframe","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:44.705191Z","iopub.execute_input":"2024-04-17T19:04:44.706400Z","iopub.status.idle":"2024-04-17T19:04:48.178170Z","shell.execute_reply.started":"2024-04-17T19:04:44.706349Z","shell.execute_reply":"2024-04-17T19:04:48.177013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converting dataframe to an array and shuffling it randomly","metadata":{}},{"cell_type":"code","source":"data = np.array(dataset) \nnp.random.shuffle(data)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:48.180427Z","iopub.execute_input":"2024-04-17T19:04:48.180784Z","iopub.status.idle":"2024-04-17T19:04:49.495104Z","shell.execute_reply.started":"2024-04-17T19:04:48.180752Z","shell.execute_reply":"2024-04-17T19:04:49.494099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Splitting Data into Development and Training Sets\nImages have 255 pixels","metadata":{}},{"cell_type":"code","source":"data_dev = data[0:1000].T #development set\ny_dev = data_dev[0]\nx_dev = data_dev[1:]\nx_dev = x_dev / 255. #scaling value from 0 to 1\n\ndata_train = data[1000:].T #training set\ny_train = data_train[0]\nx_train = data_train[1:]\nx_train = x_train / 255.\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.496342Z","iopub.execute_input":"2024-04-17T19:04:49.496712Z","iopub.status.idle":"2024-04-17T19:04:49.639456Z","shell.execute_reply.started":"2024-04-17T19:04:49.496679Z","shell.execute_reply":"2024-04-17T19:04:49.638371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Initial parameters","metadata":{}},{"cell_type":"code","source":"def init_params():\n    input_size = 784 #images are 28x28\n    hidden_size = 10\n    output_size = 10\n    w1 = np.random.randn(hidden_size, input_size) * 0.01\n    b1 = np.zeros((hidden_size, 1))\n    w2 = np.random.randn(output_size, hidden_size) * 0.01\n    b2 = np.zeros((output_size, 1))\n\n    return w1, b1, w2, b2\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.640829Z","iopub.execute_input":"2024-04-17T19:04:49.641173Z","iopub.status.idle":"2024-04-17T19:04:49.648267Z","shell.execute_reply.started":"2024-04-17T19:04:49.641141Z","shell.execute_reply":"2024-04-17T19:04:49.647067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Activation functions","metadata":{}},{"cell_type":"markdown","source":"Relu(z) = max(0, z)","metadata":{}},{"cell_type":"markdown","source":"$(z_i) = \\frac{e^{z_{i}}}{\\sum_{j=1}^K e^{z_{j}}} \\ \\ \\ $\n\n","metadata":{}},{"cell_type":"code","source":"def relu(z): #Rectified Linear Unit\n    return np.maximum(0, z)\ndef softmax(z): #Softmax function\n    expZ = np.exp(z) / sum(np.exp(z))\n    return expZ\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.652783Z","iopub.execute_input":"2024-04-17T19:04:49.653309Z","iopub.status.idle":"2024-04-17T19:04:49.661691Z","shell.execute_reply.started":"2024-04-17T19:04:49.653260Z","shell.execute_reply":"2024-04-17T19:04:49.660539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Forward propagation","metadata":{}},{"cell_type":"markdown","source":"$z^{[1]} = w^{[1]} x + b^{[1]}$\n\n$a^{[1]} = _{\\text{ReLU}}(z^{[1]}))$\n\n$z^{[2]} = w^{[2]} A^{[1]} + b^{[2]}$\n\n$a^{[2]} = _{\\text{softmax}}(z^{[2]})$\n","metadata":{}},{"cell_type":"code","source":"def forward(w1, b1, w2, b2, x):\n    z1 = np.dot(w1, x) + b1\n    a1 = relu(z1)\n    z2 = np.dot(w2, a1) + b2\n    a2 = softmax(z2)\n    return z1, a1, z2, a2\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.663192Z","iopub.execute_input":"2024-04-17T19:04:49.663641Z","iopub.status.idle":"2024-04-17T19:04:49.674279Z","shell.execute_reply.started":"2024-04-17T19:04:49.663605Z","shell.execute_reply":"2024-04-17T19:04:49.673125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converting the labels (y) into one-hot encoded vectors.","metadata":{}},{"cell_type":"code","source":"def one_hot(y):\n    one_hot = np.eye(10)[y.reshape(-1)]\n    return one_hot.T","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.676119Z","iopub.execute_input":"2024-04-17T19:04:49.676617Z","iopub.status.idle":"2024-04-17T19:04:49.685423Z","shell.execute_reply.started":"2024-04-17T19:04:49.676567Z","shell.execute_reply":"2024-04-17T19:04:49.684588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Computing loss","metadata":{}},{"cell_type":"code","source":"def compute_loss(a2, y):\n    m = y.shape[1]\n    logprobs = np.multiply(np.log(a2), y)\n    loss = - np.sum(logprobs) / m\n    return loss","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.686580Z","iopub.execute_input":"2024-04-17T19:04:49.687050Z","iopub.status.idle":"2024-04-17T19:04:49.698909Z","shell.execute_reply.started":"2024-04-17T19:04:49.686993Z","shell.execute_reply":"2024-04-17T19:04:49.698009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Implementing backward propagation to compute gradient","metadata":{}},{"cell_type":"markdown","source":"$dz^{[2]} = a^{[2]} - y$\n\n$dw^{[2]} = \\frac{1}{m} dz^{[2]} a^{[1]T}$\n\n$db^{[2]} = \\frac{1}{m} \\Sigma {dz^{[2]}}$\n\n$dz^{[1]} = w^{[2]T} dz^{[2]} .* ReLU^{[1]\\prime} (z^{[1]})$\n\n$dw^{[1]} = \\frac{1}{m} dZ^{[1]} a^{[0]T}$\n\n$db^{[1]} = \\frac{1}{m} \\Sigma {dz^{[1]}}$","metadata":{}},{"cell_type":"code","source":"def back(z1, a1, z2, a2, w2, x, y):\n    m = x.shape[1]\n    dz2 = a2 - y\n    dw2 = (1 / m) * np.dot(dz2, a1.T)\n    db2 = (1 / m) * np.sum(dz2)\n    dz1 = np.dot(w2.T, dz2) * (z1 > 0) #derivative of relu function\n    dw1 = (1 / m) * np.dot(dz1, x.T)\n    db1 = (1 / m) * np.sum(dz1)\n    return dw1, db1, dw2, db2\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.700222Z","iopub.execute_input":"2024-04-17T19:04:49.700709Z","iopub.status.idle":"2024-04-17T19:04:49.710779Z","shell.execute_reply.started":"2024-04-17T19:04:49.700661Z","shell.execute_reply":"2024-04-17T19:04:49.709746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Updating parameters","metadata":{}},{"cell_type":"code","source":"def update(w1, b1 , w2, b2, dw1, db1, dw2, db2, learning_rate):\n    w1 -= learning_rate * dw1\n    b1 -= learning_rate * db1\n    w2 -= learning_rate * dw2\n    b2 -= learning_rate * db2\n    return w1, b1, w2, b2\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.712070Z","iopub.execute_input":"2024-04-17T19:04:49.712408Z","iopub.status.idle":"2024-04-17T19:04:49.728648Z","shell.execute_reply.started":"2024-04-17T19:04:49.712378Z","shell.execute_reply":"2024-04-17T19:04:49.727601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Performing gradient descent to train the neural network parameters","metadata":{}},{"cell_type":"code","source":"def gradient_descent(x, y, iterations=1000, learning_rate=0.1): \n    w1, b1, w2, b2 = init_params() \n    for i in range (iterations): \n        z1, a1, z2, a2 = forward(w1, b1, w2, b2, x) \n        loss = compute_loss(a2,y)\n        dw1, db1, dw2, db2 = back(z1, a1, z2, a2, w2, x, y)\n        w1, b1, w2, b2 = update(w1, b1, w2, b2, dw1, db1, dw2, db2, learning_rate)\n        if i % 100 == 0:\n            print(\"Iteration:\", i)\n            print(\"Loss:\", loss)\n    return w1, b1, w2 ,b2\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.730369Z","iopub.execute_input":"2024-04-17T19:04:49.730769Z","iopub.status.idle":"2024-04-17T19:04:49.742341Z","shell.execute_reply.started":"2024-04-17T19:04:49.730732Z","shell.execute_reply":"2024-04-17T19:04:49.741369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Evaluating model","metadata":{}},{"cell_type":"code","source":"def evaluate_model(x, y, w1, b1, w2, b2):\n    _, _, _, a2 = forward(w1, b1, w2, b2, x)\n    predictions = np.argmax(a2, axis=0)\n    accuracy = np.mean(predictions == y) * 100\n    return accuracy\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.743371Z","iopub.execute_input":"2024-04-17T19:04:49.743714Z","iopub.status.idle":"2024-04-17T19:04:49.754413Z","shell.execute_reply.started":"2024-04-17T19:04:49.743685Z","shell.execute_reply":"2024-04-17T19:04:49.753394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_prediction(index, w1, b1, w2, b2):\n    current_image = x_train[:, index, None]\n    z1, a1, z2, a2 = forward(w1, b1, w2, b2, current_image)\n    prediction = np.argmax(a2)\n    label = y_train[index]\n    \n    print(\"Prediction:\", prediction)\n    print(\"Label:\", label)\n    \n    current_image = current_image.reshape((28, 28)) * 255\n    plt.gray()\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.756459Z","iopub.execute_input":"2024-04-17T19:04:49.756970Z","iopub.status.idle":"2024-04-17T19:04:49.767303Z","shell.execute_reply.started":"2024-04-17T19:04:49.756923Z","shell.execute_reply":"2024-04-17T19:04:49.765921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trained_w1, trained_b1, trained_w2, trained_b2 = gradient_descent(x_train, one_hot(y_train))","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:04:49.771724Z","iopub.execute_input":"2024-04-17T19:04:49.772193Z","iopub.status.idle":"2024-04-17T19:07:38.768461Z","shell.execute_reply.started":"2024-04-17T19:04:49.772151Z","shell.execute_reply":"2024-04-17T19:07:38.767038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_prediction(0, trained_w1, trained_b1, trained_w2, trained_b2)\ntest_prediction(1, trained_w1, trained_b1, trained_w2, trained_b2)\ntest_prediction(2, trained_w1, trained_b1, trained_w2, trained_b2)\ntest_prediction(3, trained_w1, trained_b1, trained_w2, trained_b2)\ntest_prediction(4, trained_w1, trained_b1, trained_w2, trained_b2)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:07:38.770964Z","iopub.execute_input":"2024-04-17T19:07:38.771912Z","iopub.status.idle":"2024-04-17T19:07:40.081382Z","shell.execute_reply.started":"2024-04-17T19:07:38.771854Z","shell.execute_reply":"2024-04-17T19:07:40.080192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dev_accuracy = evaluate_model(x_train, y_train, trained_w1, trained_b1, trained_w2, trained_b2)\nprint(\"Development Set Accuracy: {:.2f}%\".format(dev_accuracy))","metadata":{"execution":{"iopub.status.busy":"2024-04-17T19:07:40.083162Z","iopub.execute_input":"2024-04-17T19:07:40.083621Z","iopub.status.idle":"2024-04-17T19:07:40.195142Z","shell.execute_reply.started":"2024-04-17T19:07:40.083582Z","shell.execute_reply":"2024-04-17T19:07:40.190578Z"},"trusted":true},"execution_count":null,"outputs":[]}]}