{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":12255872,"sourceType":"datasetVersion","datasetId":7722583}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":980.119683,"end_time":"2024-07-12T06:08:35.413337","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-07-12T05:52:15.293654","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport random\nnp.random.seed(0)\nrandom.seed(0)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-07-12T07:54:42.063972Z","iopub.execute_input":"2025-07-12T07:54:42.064319Z","iopub.status.idle":"2025-07-12T07:54:43.823826Z","shell.execute_reply.started":"2025-07-12T07:54:42.064290Z","shell.execute_reply":"2025-07-12T07:54:43.819365Z"},"papermill":{"duration":1.017389,"end_time":"2024-07-12T05:52:19.550018","exception":false,"start_time":"2024-07-12T05:52:18.532629","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/handwritten-digits/train.csv')\ndata = np.array(data)\nm, n = data.shape\nnp.random.shuffle(data) \n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:54:43.826131Z","iopub.execute_input":"2025-07-12T07:54:43.826502Z","iopub.status.idle":"2025-07-12T07:54:47.294927Z","shell.execute_reply.started":"2025-07-12T07:54:43.826474Z","shell.execute_reply":"2025-07-12T07:54:47.289746Z"},"papermill":{"duration":4.931542,"end_time":"2024-07-12T05:52:24.489141","exception":false,"start_time":"2024-07-12T05:52:19.557599","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dev = data[0:28000].T\nY_dev = data_dev[0]\nX_dev = data_dev[1:n]\nX_dev = X_dev / 255.","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:54:47.297055Z","iopub.execute_input":"2025-07-12T07:54:47.297309Z","iopub.status.idle":"2025-07-12T07:54:47.375248Z","shell.execute_reply.started":"2025-07-12T07:54:47.297283Z","shell.execute_reply":"2025-07-12T07:54:47.369693Z"},"papermill":{"duration":0.094741,"end_time":"2024-07-12T05:52:24.59137","exception":false,"start_time":"2024-07-12T05:52:24.496629","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train = data[1000:m].T\nY_train = data_train[0]\nX_train = data_train[1:n]\nX_train = X_train / 255.\n_, m_train = X_train.shape","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:54:47.377838Z","iopub.execute_input":"2025-07-12T07:54:47.378127Z","iopub.status.idle":"2025-07-12T07:54:47.485886Z","shell.execute_reply.started":"2025-07-12T07:54:47.378102Z","shell.execute_reply":"2025-07-12T07:54:47.480356Z"},"papermill":{"duration":0.144098,"end_time":"2024-07-12T05:52:24.742667","exception":false,"start_time":"2024-07-12T05:52:24.598569","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def init_params():\n    W1 = np.random.rand(128, 784) - 0.5\n    b1 = np.random.rand(128, 1) - 0.5\n    W2 = np.random.rand(64, 128) - 0.5\n    b2 = np.random.rand(64, 1) - 0.5\n    W3 = np.random.rand(10, 64) - 0.5\n    b3 = np.random.rand(10, 1) - 0.5\n    return W1, b1, W2, b2, W3, b3\n\n\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:54:47.488307Z","iopub.execute_input":"2025-07-12T07:54:47.488543Z","iopub.status.idle":"2025-07-12T07:54:47.501454Z","shell.execute_reply.started":"2025-07-12T07:54:47.488513Z","shell.execute_reply":"2025-07-12T07:54:47.495335Z"},"papermill":{"duration":0.01963,"end_time":"2024-07-12T05:52:24.770126","exception":false,"start_time":"2024-07-12T05:52:24.750496","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Activation functions and their derivatives\ndef LeakyReLU(Z, alpha=0.01):\n    return np.maximum(alpha * Z, Z)\n\ndef ELU(Z, alpha=1.0):\n    return np.where(Z > 0, Z, alpha * (np.exp(Z) - 1))\n\ndef ReLU(Z):\n    return np.maximum(0, Z)\n\ndef LeakyReLU_deriv(Z, alpha=0.01):\n    return np.where(Z > 0, 1, alpha)\n\ndef ELU_deriv(Z, alpha=1.0):\n    return np.where(Z > 0, 1, alpha * np.exp(Z))\n\ndef ReLU_deriv(Z):\n    return np.where(Z > 0, 1, 0)\n\ndef softmax(Z):\n    A = np.exp(Z) / np.sum(np.exp(Z), axis=0)\n    return A\n\ndef sigmoid(Z):\n    return 1 / (1 + np.exp(-Z))\n\ndef sigmoid_deriv(Z):\n    return sigmoid(Z) * (1 - sigmoid(Z))\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:54:59.293301Z","iopub.execute_input":"2025-07-12T07:54:59.293628Z","iopub.status.idle":"2025-07-12T07:54:59.307409Z","shell.execute_reply.started":"2025-07-12T07:54:59.293602Z","shell.execute_reply":"2025-07-12T07:54:59.302663Z"},"papermill":{"duration":0.023567,"end_time":"2024-07-12T05:52:24.801112","exception":false,"start_time":"2024-07-12T05:52:24.777545","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def forward_prop(W1, b1, W2, b2, W3, b3, X, activation_hidden1, activation_hidden2):\n    Z1 = W1.dot(X) + b1\n    A1 = activation_hidden1(Z1)\n    Z2 = W2.dot(A1) + b2\n    A2 = activation_hidden2(Z2)\n    Z3 = W3.dot(A2) + b3\n    A3 = softmax(Z3)\n    return Z1, A1, Z2, A2, Z3, A3\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:55:01.022855Z","iopub.execute_input":"2025-07-12T07:55:01.023233Z","iopub.status.idle":"2025-07-12T07:55:01.036256Z","shell.execute_reply.started":"2025-07-12T07:55:01.023203Z","shell.execute_reply":"2025-07-12T07:55:01.029807Z"},"papermill":{"duration":0.018105,"end_time":"2024-07-12T05:52:24.826766","exception":false,"start_time":"2024-07-12T05:52:24.808661","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def one_hot(Y):\n    one_hot_Y = np.zeros((Y.size, Y.max() + 1))\n    one_hot_Y[np.arange(Y.size), Y] = 1\n    one_hot_Y = one_hot_Y.T\n    return one_hot_Y\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:55:01.262769Z","iopub.execute_input":"2025-07-12T07:55:01.263106Z","iopub.status.idle":"2025-07-12T07:55:01.274922Z","shell.execute_reply.started":"2025-07-12T07:55:01.263081Z","shell.execute_reply":"2025-07-12T07:55:01.269411Z"},"papermill":{"duration":0.017118,"end_time":"2024-07-12T05:52:24.85172","exception":false,"start_time":"2024-07-12T05:52:24.834602","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def backward_prop(Z1, A1, Z2, A2, Z3, A3, W1, W2, W3, X, Y, activation_hidden1_deriv, activation_hidden2_deriv):\n    one_hot_Y = one_hot(Y)\n    dZ3 = A3 - one_hot_Y\n    dW3 = 1 / m * dZ3.dot(A2.T)\n    db3 = 1 / m * np.sum(dZ3, axis=1, keepdims=True)\n    dZ2 = W3.T.dot(dZ3) * activation_hidden2_deriv(Z2)\n    dW2 = 1 / m * dZ2.dot(A1.T)\n    db2 = 1 / m * np.sum(dZ2, axis=1, keepdims=True)\n    dZ1 = W2.T.dot(dZ2) * activation_hidden1_deriv(Z1)\n    dW1 = 1 / m * dZ1.dot(X.T)\n    db1 = 1 / m * np.sum(dZ1, axis=1, keepdims=True)\n    return dW1, db1, dW2, db2, dW3, db3\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:55:02.798680Z","iopub.execute_input":"2025-07-12T07:55:02.799050Z","iopub.status.idle":"2025-07-12T07:55:02.811660Z","shell.execute_reply.started":"2025-07-12T07:55:02.799022Z","shell.execute_reply":"2025-07-12T07:55:02.806376Z"},"papermill":{"duration":0.021113,"end_time":"2024-07-12T05:52:24.880242","exception":false,"start_time":"2024-07-12T05:52:24.859129","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def update_params(W1, b1, W2, b2, W3, b3, dW1, db1, dW2, db2, dW3, db3, alpha):\n    W1 = W1 - alpha * dW1\n    b1 = b1 - alpha * db1    \n    W2 = W2 - alpha * dW2  \n    b2 = b2 - alpha * db2\n    W3 = W3 - alpha * dW3\n    b3 = b3 - alpha * db3\n    return W1, b1, W2, b2, W3, b3\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:55:03.133311Z","iopub.execute_input":"2025-07-12T07:55:03.133652Z","iopub.status.idle":"2025-07-12T07:55:03.147034Z","shell.execute_reply.started":"2025-07-12T07:55:03.133624Z","shell.execute_reply":"2025-07-12T07:55:03.141017Z"},"papermill":{"duration":0.018409,"end_time":"2024-07-12T05:52:24.906226","exception":false,"start_time":"2024-07-12T05:52:24.887817","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_accuracy(predictions, Y):\n    return np.sum(predictions == Y) / Y.size\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:55:06.373185Z","iopub.execute_input":"2025-07-12T07:55:06.373517Z","iopub.status.idle":"2025-07-12T07:55:06.385758Z","shell.execute_reply.started":"2025-07-12T07:55:06.373492Z","shell.execute_reply":"2025-07-12T07:55:06.380908Z"},"papermill":{"duration":0.016274,"end_time":"2024-07-12T05:52:24.929887","exception":false,"start_time":"2024-07-12T05:52:24.913613","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def gradient_descent(X, Y, alpha, iterations, activation_hidden1, activation_hidden2, activation_hidden1_deriv, activation_hidden2_deriv):\n    W1, b1, W2, b2, W3, b3 = init_params()\n    accuracy_list = []\n    for i in range(iterations):\n        Z1, A1, Z2, A2, Z3, A3 = forward_prop(W1, b1, W2, b2, W3, b3, X, activation_hidden1, activation_hidden2)\n        dW1, db1, dW2, db2, dW3, db3 = backward_prop(Z1, A1, Z2, A2, Z3, A3, W1, W2, W3, X, Y, activation_hidden1_deriv, activation_hidden2_deriv)\n        W1, b1, W2, b2, W3, b3 = update_params(W1, b1, W2, b2, W3, b3, dW1, db1, dW2, db2, dW3, db3, alpha)\n        if i % 10 == 0:\n            predictions = np.argmax(A3, axis=0)\n            accuracy = get_accuracy(predictions, Y)\n            accuracy_list.append(accuracy)\n            print(f\"Iteration {i}: {accuracy}\")\n    return W1, b1, W2, b2, W3, b3, accuracy_list\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:55:06.648306Z","iopub.execute_input":"2025-07-12T07:55:06.648672Z","iopub.status.idle":"2025-07-12T07:55:06.662667Z","shell.execute_reply.started":"2025-07-12T07:55:06.648644Z","shell.execute_reply":"2025-07-12T07:55:06.657674Z"},"papermill":{"duration":0.021122,"end_time":"2024-07-12T05:52:24.958217","exception":false,"start_time":"2024-07-12T05:52:24.937095","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_predictions(X, W1, b1, W2, b2, W3, b3, activation_hidden1, activation_hidden2):\n    _, _, _, _, _, A3 = forward_prop(W1, b1, W2, b2, W3, b3, X, activation_hidden1, activation_hidden2)\n    predictions = np.argmax(A3, axis=0)\n    return predictions\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:55:08.113306Z","iopub.execute_input":"2025-07-12T07:55:08.113683Z","iopub.status.idle":"2025-07-12T07:55:08.126272Z","shell.execute_reply.started":"2025-07-12T07:55:08.113658Z","shell.execute_reply":"2025-07-12T07:55:08.120611Z"},"papermill":{"duration":0.017626,"end_time":"2024-07-12T05:52:24.984159","exception":false,"start_time":"2024-07-12T05:52:24.966533","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"training with LeakyReLU and ELU\")\nW1, b1, W2, b2, W3, b3, accuracy_list_leaky_elu = gradient_descent(\n    X_dev, Y_dev, 0.10, 1000, LeakyReLU, ELU, LeakyReLU_deriv, ELU_deriv)\n\npredictions_test_leaky_elu = make_predictions(X_dev, W1, b1, W2, b2, W3, b3, LeakyReLU, ELU)\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:55:10.543699Z","iopub.execute_input":"2025-07-12T07:55:10.544054Z","iopub.status.idle":"2025-07-12T07:57:42.740018Z","shell.execute_reply.started":"2025-07-12T07:55:10.544028Z","shell.execute_reply":"2025-07-12T07:57:42.735457Z"},"papermill":{"duration":253.452178,"end_time":"2024-07-12T05:56:38.44357","exception":false,"start_time":"2024-07-12T05:52:24.991392","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training and testing with ReLU\nprint(\"training with only ReLU\")\nW1, b1, W2, b2, W3, b3, accuracy_list_relu = gradient_descent(\n    X_train, Y_train, 0.10, 1000, ReLU, ReLU, ReLU_deriv, ReLU_deriv)\n\npredictions_test_relu = make_predictions(X_dev, W1, b1, W2, b2, W3, b3, ReLU, ReLU)\n","metadata":{"execution":{"iopub.status.busy":"2025-07-12T07:57:42.744541Z","iopub.execute_input":"2025-07-12T07:57:42.744883Z","iopub.status.idle":"2025-07-12T08:00:54.635056Z","shell.execute_reply.started":"2025-07-12T07:57:42.744828Z","shell.execute_reply":"2025-07-12T08:00:54.627678Z"},"papermill":{"duration":316.027657,"end_time":"2024-07-12T06:01:54.497304","exception":false,"start_time":"2024-07-12T05:56:38.469647","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"training with Sigmoid\")\nW1, b1, W2, b2, W3, b3, accuracy_list_sigmoid = gradient_descent(\n    X_dev, Y_dev, 0.10, 1000, sigmoid, sigmoid, sigmoid_deriv, sigmoid_deriv)\n\npredictions_test_sigmoid = make_predictions(X_dev, W1, b1, W2, b2, W3, b3, sigmoid, sigmoid)\n","metadata":{"papermill":{"duration":399.54312,"end_time":"2024-07-12T06:08:34.077945","exception":false,"start_time":"2024-07-12T06:01:54.534825","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-07-12T08:00:54.638628Z","iopub.execute_input":"2025-07-12T08:00:54.639015Z","iopub.status.idle":"2025-07-12T08:04:04.210464Z","shell.execute_reply.started":"2025-07-12T08:00:54.638975Z","shell.execute_reply":"2025-07-12T08:04:04.203460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(accuracy_list_leaky_elu, label='LeakyReLU + ELU')\nplt.plot(accuracy_list_relu, label='ReLU')\nplt.plot(accuracy_list_sigmoid, label='Sigmoid')\nplt.xlabel('Iteration (x10)')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n","metadata":{"papermill":{"duration":0.32768,"end_time":"2024-07-12T06:08:34.452857","exception":false,"start_time":"2024-07-12T06:08:34.125177","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-07-12T08:04:04.214318Z","iopub.execute_input":"2025-07-12T08:04:04.214668Z","iopub.status.idle":"2025-07-12T08:04:04.472393Z","shell.execute_reply.started":"2025-07-12T08:04:04.214628Z","shell.execute_reply":"2025-07-12T08:04:04.467267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def test_prediction(index, W1, b1, W2, b2, W3, b3):\n    current_image = X_train[:, index, None]\n    prediction = make_predictions(X_train[:, index, None], W1, b1, W2, b2, W3, b3, ReLU, ELU)\n    label = Y_train[index]\n    print(\"Prediction: \", prediction)\n    print(\"Label: \", label)\n    \n    current_image = current_image.reshape((28, 28)) * 255\n    plt.gray()\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()","metadata":{"papermill":{"duration":0.035619,"end_time":"2024-07-12T06:08:34.51071","exception":false,"start_time":"2024-07-12T06:08:34.475091","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-07-12T08:05:07.213468Z","iopub.execute_input":"2025-07-12T08:05:07.213742Z","iopub.status.idle":"2025-07-12T08:05:07.226858Z","shell.execute_reply.started":"2025-07-12T08:05:07.213720Z","shell.execute_reply":"2025-07-12T08:05:07.220139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_prediction(14, W1, b1, W2, b2, W3, b3)","metadata":{"papermill":{"duration":0.285759,"end_time":"2024-07-12T06:08:34.81892","exception":false,"start_time":"2024-07-12T06:08:34.533161","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-07-12T08:05:18.913935Z","iopub.execute_input":"2025-07-12T08:05:18.914262Z","iopub.status.idle":"2025-07-12T08:05:19.035295Z","shell.execute_reply.started":"2025-07-12T08:05:18.914234Z","shell.execute_reply":"2025-07-12T08:05:19.029899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.023638,"end_time":"2024-07-12T06:08:34.866029","exception":false,"start_time":"2024-07-12T06:08:34.842391","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}