{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nmnist_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nmnist_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nmnist_train","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:43:37.987671Z","iopub.execute_input":"2025-07-08T10:43:37.987994Z","iopub.status.idle":"2025-07-08T10:43:41.782505Z","shell.execute_reply.started":"2025-07-08T10:43:37.987968Z","shell.execute_reply":"2025-07-08T10:43:41.781662Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train = mnist_train['label']\nX_train = mnist_train.iloc[: ,1:].T\nX_test = mnist_test.T\n\n# X_train = X_train / 255.\n# X_test = X_test / 255.\n\nprint(f\"X_train => {X_train.shape}\")\nprint(f\"y_train => {y_train.shape}\")\nprint(f\"X_test => {X_test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:43:45.472247Z","iopub.execute_input":"2025-07-08T10:43:45.472985Z","iopub.status.idle":"2025-07-08T10:43:45.479740Z","shell.execute_reply.started":"2025-07-08T10:43:45.472953Z","shell.execute_reply":"2025-07-08T10:43:45.479020Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Preprocessing the labels","metadata":{}},{"cell_type":"code","source":"def one_hot_encode(y):\n    m = y.shape[0]\n    one_hot_y = np.zeros((10, m))\n    one_hot_y[y, np.arange(m)] = 1\n    return one_hot_y\n\ny_train = one_hot_encode(y_train)\nprint(f\"y_train => {y_train.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:43:48.674603Z","iopub.execute_input":"2025-07-08T10:43:48.674924Z","iopub.status.idle":"2025-07-08T10:43:48.681227Z","shell.execute_reply.started":"2025-07-08T10:43:48.674897Z","shell.execute_reply":"2025-07-08T10:43:48.680467Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### The general methodology to build a Neural Network is to:\n\n1. Define the neural network structure ( # of input units,  # of hidden units, etc). \n2. Initialize the model's parameters\n3. Loop:\n    - Implement forward propagation\n    - Compute loss\n    - Implement backward propagation to get the gradients\n    - Update parameters (gradient descent)","metadata":{}},{"cell_type":"markdown","source":"## 1. Define the architecture","metadata":{}},{"cell_type":"code","source":"# Define the architecture\ndef layer_sizes(X, y):\n    n_x = X.shape[0]\n    n_h = 256\n    n_y = y.shape[0]\n    \n    return (n_x, n_h, n_y)\n\n(n_x, n_h, n_y) = layer_sizes(X_train, y_train)\nprint(\"The size of the input layer is: n_x = \" + str(n_x))\nprint(\"The size of the hidden layer is: n_h = \" + str(n_h))\nprint(\"The size of the output layer is: n_y = \" + str(n_y))","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2025-07-08T10:43:56.296711Z","iopub.execute_input":"2025-07-08T10:43:56.297101Z","iopub.status.idle":"2025-07-08T10:43:56.304442Z","shell.execute_reply.started":"2025-07-08T10:43:56.297069Z","shell.execute_reply":"2025-07-08T10:43:56.303206Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Initialize the parameters","metadata":{}},{"cell_type":"code","source":"def initialize_parameters(n_x, n_h, n_y):\n    W1 = np.random.randn(n_h, n_x) * 0.01\n    b1 = np.zeros((n_h, 1))\n    W2 = np.random.randn(n_y, n_h) * 0.01\n    b2 = np.zeros((n_y, 1))\n\n    parameters = {\"W1\": W1,\n                  \"b1\": b1,\n                  \"W2\": W2,\n                  \"b2\": b2}\n    return parameters\n\nparameters = initialize_parameters(n_x, n_h, n_y)\n\nprint(f\"W1 => {parameters['W1'].shape}\")\nprint(f\"b1 => {parameters['b1'].shape}\")\nprint(f\"W2 => {parameters['W2'].shape}\")\nprint(f\"b2 => {parameters['b2'].shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:43:58.607590Z","iopub.execute_input":"2025-07-08T10:43:58.607918Z","iopub.status.idle":"2025-07-08T10:43:58.646595Z","shell.execute_reply.started":"2025-07-08T10:43:58.607890Z","shell.execute_reply":"2025-07-08T10:43:58.645618Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.1 Implementing Forward Propagation","metadata":{}},{"cell_type":"markdown","source":"### Let's make some activation functions","metadata":{}},{"cell_type":"code","source":"def softmax(z):\n    z -= np.max(z, axis=0, keepdims=True)\n    return np.exp(z) / np.sum(np.exp(z), axis=0, keepdims=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:44:01.486220Z","iopub.execute_input":"2025-07-08T10:44:01.486616Z","iopub.status.idle":"2025-07-08T10:44:01.491615Z","shell.execute_reply.started":"2025-07-08T10:44:01.486588Z","shell.execute_reply":"2025-07-08T10:44:01.490614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ReLU(z):\n    return np.maximum(0, z)\n\ndef ReLU_derivative(z):\n    return z>0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:44:03.026439Z","iopub.execute_input":"2025-07-08T10:44:03.026774Z","iopub.status.idle":"2025-07-08T10:44:03.032117Z","shell.execute_reply.started":"2025-07-08T10:44:03.026751Z","shell.execute_reply":"2025-07-08T10:44:03.031204Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def forward_propagation(X, parameters):\n    Z1 = np.dot(parameters['W1'], X) + parameters[\"b1\"]\n    A1 = ReLU(Z1)\n    Z2 = np.dot(parameters['W2'], A1) + parameters[\"b2\"]\n    A2 = softmax(Z2)\n\n    forward_cache = {\n        \"Z1\": Z1,\n        \"A1\": A1,\n        \"Z2\": Z2,\n        \"A2\": A2,\n    }\n    return A2, forward_cache\n\nA2, forward_cache = forward_propagation(X_train, parameters)\nprint(f\"A2 => {A2.shape}\")\nprint(\"A2 = \"+ str(A2))\n#print(f\"Forward cache => {forward_cache}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:44:04.346961Z","iopub.execute_input":"2025-07-08T10:44:04.347794Z","iopub.status.idle":"2025-07-08T10:44:05.008640Z","shell.execute_reply.started":"2025-07-08T10:44:04.347763Z","shell.execute_reply":"2025-07-08T10:44:05.007660Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Make Predictions","metadata":{}},{"cell_type":"code","source":"def predict(X, parameters):\n    A2, forward_cache = forward_propagation(X, parameters)\n    y_pred = np.argmax(A2, axis=0)\n    return y_pred\n\ny_pred = predict(X_train, parameters)\ny_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T11:13:54.064132Z","iopub.execute_input":"2025-07-08T11:13:54.064426Z","iopub.status.idle":"2025-07-08T11:13:54.852315Z","shell.execute_reply.started":"2025-07-08T11:13:54.064405Z","shell.execute_reply":"2025-07-08T11:13:54.851454Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2 Computing Cost and Accuracy","metadata":{}},{"cell_type":"code","source":"def compute_cost(A2, y):\n    m = y.shape[1]\n    epsilon = 1e-8\n    cost = -np.sum(y * np.log(A2 + epsilon)) / m\n    return np.squeeze(cost)\n\ndef compute_accuracy(y_pred, y_true):\n    labels = np.argmax(y_true, axis=0)\n    return np.mean(y_pred == labels) * 100\n\ncompute_accuracy(y_pred, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:44:09.892290Z","iopub.execute_input":"2025-07-08T10:44:09.892685Z","iopub.status.idle":"2025-07-08T10:44:09.903622Z","shell.execute_reply.started":"2025-07-08T10:44:09.892657Z","shell.execute_reply":"2025-07-08T10:44:09.902576Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.3 Implementing Backward Propagation","metadata":{}},{"cell_type":"code","source":"def backward_propagation(parameters, forward_cache, X, Y):\n    W1 = parameters[\"W1\"]\n    W2 = parameters[\"W2\"]\n\n    Z1 = forward_cache[\"Z1\"]\n    A1 = forward_cache[\"A1\"]\n    A2 = forward_cache[\"A2\"]\n\n    m = X.shape[1]\n\n    dZ2 = A2 - Y\n    dW2 = np.dot(dZ2, A1.T) / m\n    db2 = np.sum(dZ2, axis=1, keepdims=True) / m\n    dZ1 = np.dot(W2.T, dZ2) * ReLU_derivative(Z1)\n    dW1 = np.dot(dZ1, X.T) / m\n    db1 = np.sum(dZ1, axis=1, keepdims=True)\n\n    grads = {\n        \"dW2\": dW2,\n        \"db2\": db2,\n        \"dW1\": dW1,\n        \"db1\": db1\n    }\n    return grads\n\ngrads = backward_propagation(parameters, forward_cache, X_train, y_train)\nprint (\"dW2 => \", grads[\"dW2\"].shape)\nprint (\"db2 => \", grads[\"db2\"].shape)\nprint (\"dW1 => \", grads[\"dW1\"].shape)\nprint (\"db1 => \", grads[\"db1\"].shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:44:13.300210Z","iopub.execute_input":"2025-07-08T10:44:13.301174Z","iopub.status.idle":"2025-07-08T10:44:13.757939Z","shell.execute_reply.started":"2025-07-08T10:44:13.301145Z","shell.execute_reply":"2025-07-08T10:44:13.757091Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.4 Updating Parameters","metadata":{}},{"cell_type":"code","source":"def update_parameters(parameters, grads, learning_rate=0.1):\n    parameters[\"W1\"] -= learning_rate * grads[\"dW1\"]\n    parameters[\"b1\"] -= learning_rate * grads[\"db1\"]\n    parameters[\"W2\"] -= learning_rate * grads[\"dW2\"]\n    parameters[\"b2\"] -= learning_rate * grads[\"db2\"]\n    return parameters\n\nparameters = update_parameters(parameters, grads)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:44:15.543317Z","iopub.execute_input":"2025-07-08T10:44:15.543642Z","iopub.status.idle":"2025-07-08T10:44:15.550064Z","shell.execute_reply.started":"2025-07-08T10:44:15.543618Z","shell.execute_reply":"2025-07-08T10:44:15.549092Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Putting all pieces together","metadata":{}},{"cell_type":"code","source":"def nn_model(X, y, n_h, iterations=30, learning_rate=0.1):\n    np.random.seed(42)\n    \n    n_x = layer_sizes(X, y)[0]\n    n_y = layer_sizes(X, y)[2]\n\n    parameters = initialize_parameters(n_x, n_h, n_y)\n    for i in range(iterations):\n        # Forward Propagation\n        A2, forward_cache = forward_propagation(X, parameters)\n        # Calculate Predictions\n        y_pred = predict(X_train, parameters)\n        # Calculate Accuracy\n        accuracy = compute_accuracy(y_pred, y_train)\n        # Calculate cost\n        cost = compute_cost(A2, y)\n        # Backward Propagation\n        grads = backward_propagation(parameters, forward_cache, X, y)\n        # Gradient Descent\n        parameters = update_parameters(parameters, grads, learning_rate=learning_rate)\n        if (i%100 == 0):\n            print(f\"Epoch {i}/{iterations}\")\n            print(f\" categorical cross entropy: {cost} - accuracy = {accuracy}\")\n    print(f\"Final Cost: {cost} - Training Accuracy: {accuracy}\")\n    return parameters\n\nfinal_parameters = nn_model(X_train, y_train, 256, iterations=1200, learning_rate=0.0024)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T10:44:31.862329Z","iopub.execute_input":"2025-07-08T10:44:31.862699Z","iopub.status.idle":"2025-07-08T11:11:52.861676Z","shell.execute_reply.started":"2025-07-08T10:44:31.862674Z","shell.execute_reply":"2025-07-08T11:11:52.860838Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Final Submission","metadata":{}},{"cell_type":"code","source":"y_pred_test = predict(X_test, final_parameters)\nprint(\"X_test shape:\", X_test.shape)\nprint(\"y_pred_test shape:\", y_pred_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T11:35:17.012502Z","iopub.execute_input":"2025-07-08T11:35:17.012906Z","iopub.status.idle":"2025-07-08T11:35:17.405351Z","shell.execute_reply.started":"2025-07-08T11:35:17.012879Z","shell.execute_reply":"2025-07-08T11:35:17.404434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\"ImageId\": range(1, (X_test.shape[1])+1), \"Label\": y_pred_test})\nsubmission.to_csv(\"aak_mnist_from_scratch\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T11:38:44.381793Z","iopub.execute_input":"2025-07-08T11:38:44.382155Z","iopub.status.idle":"2025-07-08T11:38:44.421429Z","shell.execute_reply.started":"2025-07-08T11:38:44.382127Z","shell.execute_reply":"2025-07-08T11:38:44.420492Z"}},"outputs":[],"execution_count":null}]}