{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries and Read Datasets","metadata":{"id":"rI5gqoZCZVrJ"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"id":"BBgymHDLmx8D","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:44.918556Z","iopub.execute_input":"2025-02-24T17:19:44.918992Z","iopub.status.idle":"2025-02-24T17:19:44.924199Z","shell.execute_reply.started":"2025-02-24T17:19:44.918949Z","shell.execute_reply":"2025-02-24T17:19:44.922845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"id":"DBkthk72YXe5","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:44.926252Z","iopub.execute_input":"2025-02-24T17:19:44.926763Z","iopub.status.idle":"2025-02-24T17:19:49.075966Z","shell.execute_reply.started":"2025-02-24T17:19:44.926710Z","shell.execute_reply":"2025-02-24T17:19:49.074991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"id":"NbmM7deQYgsR","outputId":"9956f5cd-3a51-4b5d-c46d-6b61ed24b592","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:49.077811Z","iopub.execute_input":"2025-02-24T17:19:49.078178Z","iopub.status.idle":"2025-02-24T17:19:49.093527Z","shell.execute_reply.started":"2025-02-24T17:19:49.078151Z","shell.execute_reply":"2025-02-24T17:19:49.092431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info()","metadata":{"id":"OXiNpcyUZf3D","outputId":"c2dfbf58-0ed0-406e-8bc4-c330ffcfe299","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:49.095349Z","iopub.execute_input":"2025-02-24T17:19:49.095755Z","iopub.status.idle":"2025-02-24T17:19:49.149064Z","shell.execute_reply.started":"2025-02-24T17:19:49.095724Z","shell.execute_reply":"2025-02-24T17:19:49.147647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.describe()","metadata":{"id":"G1Om8UY5ZhTF","outputId":"ac9ade4d-a87e-4d82-bed2-9bf71e17d110","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:49.150470Z","iopub.execute_input":"2025-02-24T17:19:49.150837Z","iopub.status.idle":"2025-02-24T17:19:51.442615Z","shell.execute_reply.started":"2025-02-24T17:19:49.150799Z","shell.execute_reply":"2025-02-24T17:19:51.441598Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Check for Missing Data","metadata":{"id":"Ai7EArpHZtmv"}},{"cell_type":"code","source":"# Check if there is any missing information in the training dataset.\nhas_missing = any(df.isnull().sum().tolist())\nprint(f\"\\nDoes the training data have any missing values? {has_missing}\")\n# Note: The dataset does not contain any null values.","metadata":{"id":"5ZBPy1y9ZjlT","outputId":"fbd4ccc0-c917-4604-aa49-84812f69c839","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:51.443587Z","iopub.execute_input":"2025-02-24T17:19:51.443958Z","iopub.status.idle":"2025-02-24T17:19:51.488293Z","shell.execute_reply.started":"2025-02-24T17:19:51.443930Z","shell.execute_reply":"2025-02-24T17:19:51.486994Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualize a Sample Image from the Dataset","metadata":{"id":"Y-nX2ajnYogW"}},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"id":"b7OljMW_YiGX","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:51.491806Z","iopub.execute_input":"2025-02-24T17:19:51.492199Z","iopub.status.idle":"2025-02-24T17:19:51.497561Z","shell.execute_reply.started":"2025-02-24T17:19:51.492171Z","shell.execute_reply":"2025-02-24T17:19:51.495983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The dataset contains 784 pixel values per image (28x28 pixels).\n# Here we select the 11th record (index 10) and reshape it to display the image.\nimage = df.iloc[10][1:].to_numpy().reshape(28, 28)\nimage_label = df.iloc[10][0]","metadata":{"id":"5PT0GEDAYwon","outputId":"6d19ddf1-318f-44cc-bc95-10ecbcbbc8ae","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:51.499503Z","iopub.execute_input":"2025-02-24T17:19:51.499834Z","iopub.status.idle":"2025-02-24T17:19:51.521745Z","shell.execute_reply.started":"2025-02-24T17:19:51.499780Z","shell.execute_reply":"2025-02-24T17:19:51.520440Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(image, cmap='gray')\nplt.title(f\"Label: {image_label}\")\nplt.axis('off')\nplt.show()","metadata":{"id":"0Xavw2kLZNL7","outputId":"9fccb4b1-5d13-4506-ad1a-bf34e6fdfb58","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:51.522952Z","iopub.execute_input":"2025-02-24T17:19:51.523316Z","iopub.status.idle":"2025-02-24T17:19:51.641783Z","shell.execute_reply.started":"2025-02-24T17:19:51.523269Z","shell.execute_reply":"2025-02-24T17:19:51.640625Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now Let's build Neural Network:\n- `From Scratch` => Detailed\n- `Using Keras Library` => Easy","metadata":{"id":"djGKeE7hZ0qN"}},{"cell_type":"markdown","source":"# Neural Network From Scratch - Setup and Data Preparation","metadata":{"id":"8-_7VI67d_6P"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder","metadata":{"id":"us_cz0YOZ0cm","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:51.642903Z","iopub.execute_input":"2025-02-24T17:19:51.643351Z","iopub.status.idle":"2025-02-24T17:19:51.647879Z","shell.execute_reply.started":"2025-02-24T17:19:51.643285Z","shell.execute_reply":"2025-02-24T17:19:51.646780Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load dataset again (if running this cell independently)\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\n# Extract features (pixel values) and labels from the dataset.\n# Normalize pixel values to [0, 1] by dividing by 255.\nX = df.iloc[:, 1:].values.astype(np.float32) / 255.0\ny = df.iloc[:, 0].values.astype(np.int64)","metadata":{"id":"ZVpnWxs7eD_d","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:51.648930Z","iopub.execute_input":"2025-02-24T17:19:51.649331Z","iopub.status.idle":"2025-02-24T17:19:55.862125Z","shell.execute_reply.started":"2025-02-24T17:19:51.649274Z","shell.execute_reply":"2025-02-24T17:19:55.861046Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Now let's make one hot encoding for labels","metadata":{"id":"5jH9hJFcfORy"}},{"cell_type":"markdown","source":"`why?`\n\n- Because when we use numbers like (1 or 2 or 3 and so on), it misleads the model and makes it understand that the probability of `y = 3` is more like `2` than `8` for example. But in the real image the number `3` is more similar to `8` than `2` so it may mislead the model.\n\n**So we need to make one-hot encoding**","metadata":{}},{"cell_type":"code","source":"# One-hot encode labels\nencoder = OneHotEncoder(sparse_output=False) # for memory effiecent\ny_onehot = encoder.fit_transform(y.reshape(-1, 1))","metadata":{"id":"yosCLx8ceEmA","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:55.863323Z","iopub.execute_input":"2025-02-24T17:19:55.863697Z","iopub.status.idle":"2025-02-24T17:19:55.879287Z","shell.execute_reply.started":"2025-02-24T17:19:55.863652Z","shell.execute_reply":"2025-02-24T17:19:55.878274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train-Test Split\nX_train, X_test, y_train, y_test = train_test_split(X, y_onehot, test_size=0.2, random_state=42)\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)","metadata":{"id":"dhktcMoDeEgv","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:55.880754Z","iopub.execute_input":"2025-02-24T17:19:55.881158Z","iopub.status.idle":"2025-02-24T17:19:56.321168Z","shell.execute_reply.started":"2025-02-24T17:19:55.881101Z","shell.execute_reply":"2025-02-24T17:19:56.320013Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Neural Network From Scratch - Define Hyperparameters & Initialize Parameters","metadata":{"id":"LvlM0sfLgiEu"}},{"cell_type":"code","source":"# Define network architecture and hyperparameters.\ninput_size   = 784     # Each image is 28x28 pixels\nhidden_size1 = 128     # First hidden layer neurons\nhidden_size2 = 64      # Second hidden layer neurons\noutput_size  = 10      # 10 classes for digits 0-9\n\nalpha   = 0.001        # Learning rate\nbeta1   = 0.9          # Adam optimizer parameter\nbeta2   = 0.999        # Adam optimizer parameter\nepsilon = 1e-8         # Small constant to prevent division by zero\n\n# Xavier (He) initialization for weights and zero initialization for biases.\nnp.random.seed(42)\nW1 = np.random.randn(input_size, hidden_size1) * np.sqrt(2.0 / input_size)\nb1 = np.zeros((1, hidden_size1))\nW2 = np.random.randn(hidden_size1, hidden_size2) * np.sqrt(2.0 / hidden_size1)\nb2 = np.zeros((1, hidden_size2))\nW3 = np.random.randn(hidden_size2, output_size) * np.sqrt(2.0 / hidden_size2)\nb3 = np.zeros((1, output_size))\n\n# Initialize Adam optimizer variables (moving averages and variances) for each layer.\nmoving_avg_W1, moving_var_W1 = np.zeros_like(W1), np.zeros_like(W1)\nmoving_avg_b1, moving_var_b1 = np.zeros_like(b1), np.zeros_like(b1)\nmoving_avg_W2, moving_var_W2 = np.zeros_like(W2), np.zeros_like(W2)\nmoving_avg_b2, moving_var_b2 = np.zeros_like(b2), np.zeros_like(b2)\nmoving_avg_W3, moving_var_W3 = np.zeros_like(W3), np.zeros_like(W3)\nmoving_avg_b3, moving_var_b3 = np.zeros_like(b3), np.zeros_like(b3)","metadata":{"id":"tiQU618oeEcQ","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.322290Z","iopub.execute_input":"2025-02-24T17:19:56.322716Z","iopub.status.idle":"2025-02-24T17:19:56.336353Z","shell.execute_reply.started":"2025-02-24T17:19:56.322680Z","shell.execute_reply":"2025-02-24T17:19:56.335158Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define Activation Functions and Loss Function","metadata":{"id":"plPOhjFpipdi"}},{"cell_type":"markdown","source":"$$\nf(x) = \\max(0, x)\n$$","metadata":{"id":"AWv3pPZcig0Q"}},{"cell_type":"code","source":"def relu(Z):\n  \"\"\"ReLU activation function.\"\"\"\n  return np.maximum(0, Z)","metadata":{"id":"AY9Tw6hBeEZ6","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.337614Z","iopub.execute_input":"2025-02-24T17:19:56.338047Z","iopub.status.idle":"2025-02-24T17:19:56.345264Z","shell.execute_reply.started":"2025-02-24T17:19:56.338007Z","shell.execute_reply":"2025-02-24T17:19:56.344292Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$\nS_i = \\frac{e^{z_i}}{\\sum_{j} e^{z_j}}\n$$","metadata":{"id":"XYKAYz0phNj0"}},{"cell_type":"code","source":"def softmax(Z):\n  \"\"\"Softmax activation function for multi-class output.\"\"\"\n  exp_Z = np.exp(Z)\n  return exp_Z / np.sum(exp_Z, axis=1, keepdims=True) # Keepdims => to ensure that it will keep the same dimensions","metadata":{"id":"eZLnu0SaeEXz","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.346279Z","iopub.execute_input":"2025-02-24T17:19:56.346825Z","iopub.status.idle":"2025-02-24T17:19:56.368511Z","shell.execute_reply.started":"2025-02-24T17:19:56.346787Z","shell.execute_reply":"2025-02-24T17:19:56.367374Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Compute loss","metadata":{"id":"VyC7lZf3jc93"}},{"cell_type":"markdown","source":"$$\n\\mathcal{L} = -\\frac{1}{m} \\sum_{i=1}^{m} \\sum_{j=1}^{C} y_{ij} \\log(\\hat{y}_{ij} + \\epsilon)\n$$\n","metadata":{"id":"j-UYMJavkFgq"}},{"cell_type":"code","source":"def compute_loss(Y, Y_hat):\n  \"\"\"Compute cross-entropy loss with epsilon smoothing.\"\"\"\n  return -np.mean(Y * np.log(Y_hat + epsilon)) # to avoid infinte values from 'log(0)'","metadata":{"id":"tkViCIc9eEVV","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.369842Z","iopub.execute_input":"2025-02-24T17:19:56.370280Z","iopub.status.idle":"2025-02-24T17:19:56.388078Z","shell.execute_reply.started":"2025-02-24T17:19:56.370242Z","shell.execute_reply":"2025-02-24T17:19:56.386800Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Forward Propagation Function","metadata":{"id":"kBHuLQSZkZYu"}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"1. **First Layer:**\n   $$\n   Z_1 = X W_1 + b_1\n   $$\n   $$\n   A_1 = \\text{ReLU}(Z_1)\n   $$\n\n2. **Second Layer:**\n   $$\n   Z_2 = A_1 W_2 + b_2\n   $$\n   $$\n   A_2 = \\text{ReLU}(Z_2)\n   $$\n\n3. **Third Layer (Output Layer):**\n   $$\n   Z_3 = A_2 W_3 + b_3\n   $$\n   $$\n   A_3 = \\text{Softmax}(Z_3)\n   $$","metadata":{"id":"kfOGNCgokmtB"}},{"cell_type":"code","source":"def forward_propagation(X):\n    \"\"\"\n    Perform forward propagation through the network.\n    Returns:\n      Z1, A1, Z2, A2, Z3, A3: Intermediate computations for backpropagation.\n    \"\"\"\n    Z1 = X @ W1 + b1\n    A1 = relu(Z1)\n\n    Z2 = A1 @ W2 + b2\n    A2 = relu(Z2)\n\n    Z3 = A2 @ W3 + b3\n    A3 = softmax(Z3)\n\n    return Z1, A1, Z2, A2, Z3, A3","metadata":{"id":"nU0aRRBReETH","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.389119Z","iopub.execute_input":"2025-02-24T17:19:56.389474Z","iopub.status.idle":"2025-02-24T17:19:56.411540Z","shell.execute_reply.started":"2025-02-24T17:19:56.389446Z","shell.execute_reply":"2025-02-24T17:19:56.410386Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Backward Propagation Function","metadata":{"id":"8emhRPC6ka1F"}},{"cell_type":"markdown","source":"1. **Compute Output Layer Gradients:**\n   $$\n   dZ_3 = A_3 - Y\n   $$\n   $$\n   dW_3 = \\frac{1}{m} A_2^T dZ_3\n   $$\n   $$\n   db_3 = \\frac{1}{m} \\sum dZ_3\n   $$\n\n2. **Compute Hidden Layer 2 Gradients:**\n   $$\n   dA_2 = dZ_3 W_3^T\n   $$\n   $$\n   dZ_2 = dA_2 \\cdot \\text{ReLU}'(Z_2)\n   $$\n   $$\n   dW_2 = \\frac{1}{m} A_1^T dZ_2\n   $$\n   $$\n   db_2 = \\frac{1}{m} \\sum dZ_2\n   $$\n\n3. **Compute Hidden Layer 1 Gradients:**\n   $$\n   dA_1 = dZ_2 W_2^T\n   $$\n   $$\n   dZ_1 = dA_1 \\cdot \\text{ReLU}'(Z_1)\n   $$\n   $$\n   dW_1 = \\frac{1}{m} X^T dZ_1\n   $$\n   $$\n   db_1 = \\frac{1}{m} \\sum dZ_1\n   $$","metadata":{"id":"boKuxsN3lXPf"}},{"cell_type":"code","source":"def backward_propagation(X, Y, Z1, A1, Z2, A2, Z3, A3):\n    \"\"\"\n    Perform backward propagation to compute gradients for all parameters.\n    Returns:\n      Gradients for W1, b1, W2, b2, W3, b3.\n    \"\"\"\n    m = X.shape[0]  # number of samples in the batch\n\n    # Output layer gradients\n    dZ3 = A3 - Y\n    dW3 = (A2.T @ dZ3) / m\n    db3 = np.sum(dZ3, axis=0, keepdims=True) / m\n\n    # Second hidden layer gradients\n    dA2 = dZ3 @ W3.T\n    dZ2 = dA2 * (Z2 > 0)  # derivative of ReLU\n    dW2 = (A1.T @ dZ2) / m\n    db2 = np.sum(dZ2, axis=0, keepdims=True) / m\n\n    # First hidden layer gradients\n    dA1 = dZ2 @ W2.T\n    dZ1 = dA1 * (Z1 > 0)  # derivative of ReLU\n    dW1 = (X.T @ dZ1) / m\n    db1 = np.sum(dZ1, axis=0, keepdims=True) / m\n\n    return dW1, db1, dW2, db2, dW3, db3","metadata":{"id":"QtM51ekLjbbP","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.415785Z","iopub.execute_input":"2025-02-24T17:19:56.416159Z","iopub.status.idle":"2025-02-24T17:19:56.432420Z","shell.execute_reply.started":"2025-02-24T17:19:56.416122Z","shell.execute_reply":"2025-02-24T17:19:56.431291Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Adam Optimizer Update Function","metadata":{"id":"7a1tbkDJk6zM"}},{"cell_type":"markdown","source":"1. **Compute Moving Averages for Weight Gradients:**\n   $$\n   m_W = \\beta_1 m_W + (1 - \\beta_1) \\cdot \\nabla W\n   $$\n   $$\n   v_W = \\beta_2 v_W + (1 - \\beta_2) \\cdot (\\nabla W)^2\n   $$\n\n2. **Compute Moving Averages for Bias Gradients:**\n   $$\n   m_b = \\beta_1 m_b + (1 - \\beta_1) \\cdot \\nabla b\n   $$\n   $$\n   v_b = \\beta_2 v_b + (1 - \\beta_2) \\cdot (\\nabla b)^2\n   $$\n\n3. **Apply Bias Correction:**\n   $$\n   \\hat{m}_W = \\frac{m_W}{1 - \\beta_1^t}, \\quad \\hat{v}_W = \\frac{v_W}{1 - \\beta_2^t}\n   $$\n   $$\n   \\hat{m}_b = \\frac{m_b}{1 - \\beta_1^t}, \\quad \\hat{v}_b = \\frac{v_b}{1 - \\beta_2^t}\n   $$\n\n4. **Compute Weight and Bias Updates:**\n   $$\n   \\Delta W = \\alpha \\cdot \\frac{\\hat{m}_W}{\\sqrt{\\hat{v}_W} + \\epsilon}\n   $$\n   $$\n   \\Delta b = \\alpha \\cdot \\frac{\\hat{m}_b}{\\sqrt{\\hat{v}_b} + \\epsilon}\n   $$","metadata":{"id":"go0hzVJClDVc"}},{"cell_type":"code","source":"def adam_update(time_step, weight_gradient, bias_gradient, moving_avg_weight, moving_var_weight, moving_avg_bias, moving_var_bias):\n    \"\"\"\n    Update parameters using the Adam optimization algorithm.\n    Returns:\n      weight_update, bias_update and updated moving averages and variances.\n    \"\"\"\n    # Compute moving averages for weight gradients\n    moving_avg_weight = beta1 * moving_avg_weight + (1 - beta1) * weight_gradient\n    moving_var_weight = beta2 * moving_var_weight + (1 - beta2) * (weight_gradient ** 2)\n\n    # Compute moving averages for bias gradients\n    moving_avg_bias = beta1 * moving_avg_bias + (1 - beta1) * bias_gradient\n    moving_var_bias = beta2 * moving_var_bias + (1 - beta2) * (bias_gradient ** 2)\n\n    # Apply weight and bias correction\n    corrected_moving_avg_weight = moving_avg_weight / (1 - beta1 ** time_step)\n    corrected_moving_var_weight = moving_var_weight / (1 - beta2 ** time_step)\n\n    corrected_moving_avg_bias = moving_avg_bias / (1 - beta1 ** time_step)\n    corrected_moving_var_bias = moving_var_bias / (1 - beta2 ** time_step)\n\n    # Compute weight and bias updates\n    weight_update = alpha * corrected_moving_avg_weight / (np.sqrt(corrected_moving_var_weight) + epsilon)\n    bias_update = alpha * corrected_moving_avg_bias / (np.sqrt(corrected_moving_var_bias) + epsilon)\n\n    return weight_update, bias_update, moving_avg_weight, moving_var_weight, moving_avg_bias, moving_var_bias","metadata":{"id":"P1kBcFxHjbZM","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.434118Z","iopub.execute_input":"2025-02-24T17:19:56.434454Z","iopub.status.idle":"2025-02-24T17:19:56.452907Z","shell.execute_reply.started":"2025-02-24T17:19:56.434427Z","shell.execute_reply":"2025-02-24T17:19:56.451886Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training the Neural Network From Scratch","metadata":{"id":"Y079-xNaoK4V"}},{"cell_type":"code","source":"epochs = 10\nbatch_size = 32","metadata":{"id":"HM9Gvzs0jbWt","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.454120Z","iopub.execute_input":"2025-02-24T17:19:56.454467Z","iopub.status.idle":"2025-02-24T17:19:56.474172Z","shell.execute_reply.started":"2025-02-24T17:19:56.454433Z","shell.execute_reply":"2025-02-24T17:19:56.472922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for epoch in range(epochs):\n  for i in range(0, X_train.shape[0], batch_size):\n    X_batch = X_train[i: i+batch_size]\n    Y_batch = y_train[i: i+batch_size]\n\n    # Forward pass\n    Z1, A1, Z2, A2, Z3, A3 = forward_propagation(X_batch)\n\n    # Compute gradients\n    dW1, db1, dW2, db2, dW3, db3 = backward_propagation(X_batch, Y_batch, Z1, A1, Z2, A2, Z3, A3)\n\n    # Update weights using Adam optimizer\n    W1_update, b1_update, moving_avg_W1, moving_var_W1, moving_avg_b1, moving_var_b1 = adam_update(epoch + 1, dW1, db1, moving_avg_W1, moving_var_W1, moving_avg_b1, moving_var_b1)\n\n    W2_update, b2_update, moving_avg_W2, moving_var_W2, moving_avg_b2, moving_var_b2 = adam_update(epoch + 1, dW2, db2, moving_avg_W2, moving_var_W2, moving_avg_b2, moving_var_b2)\n\n    W3_update, b3_update, moving_avg_W3, moving_var_W3, moving_avg_b3, moving_var_b3 = adam_update(epoch + 1, dW3, db3, moving_avg_W3, moving_var_W3, moving_avg_b3, moving_var_b3)\n\n    # Apply updates\n    W1 -= W1_update\n    b1 -= b1_update\n    W2 -= W2_update\n    b2 -= b2_update\n    W3 -= W3_update\n    b3 -= b3_update\n\n  # Evaluate loss after each epoch\n  _, _, _, _, _, A3 = forward_propagation(X_train)\n  loss = compute_loss(y_train, A3)\n  print(f\"Epoch {epoch+1}/{epochs}, Loss: {loss:.4f}\")","metadata":{"id":"FpkUE2k_jbU2","outputId":"a3cfc03d-bf62-42d0-b270-e3e221f7aeb8","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:19:56.475480Z","iopub.execute_input":"2025-02-24T17:19:56.475818Z","iopub.status.idle":"2025-02-24T17:20:21.786006Z","shell.execute_reply.started":"2025-02-24T17:19:56.475791Z","shell.execute_reply":"2025-02-24T17:20:21.784894Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluate the Custom Neural Network","metadata":{"id":"LPdl25CmpgG2"}},{"cell_type":"code","source":"# Forward propagation on test set to obtain predictions.\n_, _, _, _, _, A3_test = forward_propagation(X_test)\npredictions = np.argmax(A3_test, axis=1)\nlabels = np.argmax(y_test, axis=1)\naccuracy = np.mean(predictions == labels)\nprint(f\"Test Accuracy: {accuracy * 100:.2f}%\")","metadata":{"id":"opkUkoBvjbS9","outputId":"806f4ca8-8e20-429e-a7f7-95a811ce8ab8","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:21.786992Z","iopub.execute_input":"2025-02-24T17:20:21.787265Z","iopub.status.idle":"2025-02-24T17:20:21.850526Z","shell.execute_reply.started":"2025-02-24T17:20:21.787241Z","shell.execute_reply":"2025-02-24T17:20:21.849561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize predictions on validation set\nn_images = 10\nplt.figure(figsize=(12, 6))\nfor i in range(n_images):\n    plt.subplot(1, n_images, i + 1)\n    image = X_val[i].reshape(28, 28)\n\n    # Predict for a single sample (reshaped to match input dimensions)\n    _, _, _, _, _, A3_val = forward_propagation(X_val[i].reshape(1, -1))\n    predicted_label = np.argmax(A3_val, axis=1)[0]\n    true_label = np.argmax(y_val[i])\n\n    plt.imshow(image, cmap='gray')\n    plt.axis('off')\n    plt.title(f\"True: {true_label}\\nPred: {predicted_label}\")\nplt.tight_layout()\nplt.show()","metadata":{"id":"EEo8dmV_p_DK","outputId":"cd6d0d40-4032-4040-f479-e3bd49d0ac63","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:21.851371Z","iopub.execute_input":"2025-02-24T17:20:21.851670Z","iopub.status.idle":"2025-02-24T17:20:22.471385Z","shell.execute_reply.started":"2025-02-24T17:20:21.851645Z","shell.execute_reply":"2025-02-24T17:20:22.470293Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Neural Network Using Keras - Data Preparation","metadata":{"id":"PbE9NZdgaSm8"}},{"cell_type":"code","source":"# Extract Features (Pixels) and Labels\nX = df.iloc[:,1:].values.astype('float32') # Convert features to float32\ny = df.iloc[:,0].values.astype('int64') # Convert labels to int64","metadata":{"id":"mxNQAwt6ZPLq","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:22.472467Z","iopub.execute_input":"2025-02-24T17:20:22.472753Z","iopub.status.idle":"2025-02-24T17:20:22.542533Z","shell.execute_reply.started":"2025-02-24T17:20:22.472726Z","shell.execute_reply":"2025-02-24T17:20:22.541416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train-Test Split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Train-Validation Split\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)","metadata":{"id":"-IbDylYAaoBB","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:22.543666Z","iopub.execute_input":"2025-02-24T17:20:22.544025Z","iopub.status.idle":"2025-02-24T17:20:23.170487Z","shell.execute_reply.started":"2025-02-24T17:20:22.543998Z","shell.execute_reply":"2025-02-24T17:20:23.168720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check shapes\nprint(f\"X_train shape: {X_train.shape}, y_train shape: {y_train.shape}\")\nprint(f\"X_val shape: {X_val.shape}, y_val shape: {y_val.shape}\")","metadata":{"id":"1eAOXFdKa5sy","outputId":"1ec0eeb7-8614-4758-f45a-817dc086e103","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:23.172011Z","iopub.execute_input":"2025-02-24T17:20:23.172496Z","iopub.status.idle":"2025-02-24T17:20:23.179899Z","shell.execute_reply.started":"2025-02-24T17:20:23.172459Z","shell.execute_reply":"2025-02-24T17:20:23.178630Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize Data (Scale pixel values to [0,1])\nX_train, X_val, X_test = X_train / 255.0, X_val / 255.0, X_test / 255.0","metadata":{"id":"bJGU2saxa7yZ","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:23.181165Z","iopub.execute_input":"2025-02-24T17:20:23.181531Z","iopub.status.idle":"2025-02-24T17:20:23.246355Z","shell.execute_reply.started":"2025-02-24T17:20:23.181501Z","shell.execute_reply":"2025-02-24T17:20:23.244968Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Build and Compile the Keras Model","metadata":{"id":"1-R17oGXciDu"}},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"id":"2INorfAubSnq","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:23.247494Z","iopub.execute_input":"2025-02-24T17:20:23.247795Z","iopub.status.idle":"2025-02-24T17:20:41.311540Z","shell.execute_reply.started":"2025-02-24T17:20:23.247768Z","shell.execute_reply":"2025-02-24T17:20:41.310262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Dense(128, activation='relu', input_shape = (784,)), # intiallize the input shape and first layer of neural network contain 128 neuron\n    tf.keras.layers.Dropout(0.2), # Dropout to avoid \"Over Fitting\"\n    tf.keras.layers.Dense(64, activation='relu'), # second Layer contain 64 neuron\n    tf.keras.layers.Dense(10, activation='softmax') # final layer has 10 neuran (because we have 'y' has 10 different values)\n])","metadata":{"id":"gqEH55GIbQKR","outputId":"6b1eb928-c05a-4233-f1f3-693f2741904c","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:41.312715Z","iopub.execute_input":"2025-02-24T17:20:41.313618Z","iopub.status.idle":"2025-02-24T17:20:41.505709Z","shell.execute_reply.started":"2025-02-24T17:20:41.313571Z","shell.execute_reply":"2025-02-24T17:20:41.504448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])","metadata":{"id":"p7tNHVa2cHZQ","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:41.506761Z","iopub.execute_input":"2025-02-24T17:20:41.507104Z","iopub.status.idle":"2025-02-24T17:20:41.525047Z","shell.execute_reply.started":"2025-02-24T17:20:41.507054Z","shell.execute_reply":"2025-02-24T17:20:41.523903Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train the Keras Model","metadata":{"id":"lMa4Qd1IC-uA"}},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=10, validation_data=(X_val, y_val))","metadata":{"id":"JAHHU-hxcTNO","outputId":"b1ab0271-c503-4ae4-9a06-4312c4205286","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:20:41.526352Z","iopub.execute_input":"2025-02-24T17:20:41.526727Z","iopub.status.idle":"2025-02-24T17:21:16.167886Z","shell.execute_reply.started":"2025-02-24T17:20:41.526691Z","shell.execute_reply":"2025-02-24T17:21:16.166173Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluate and Visualize Keras Model Predictions","metadata":{"id":"lQ8Icw0xckqi"}},{"cell_type":"code","source":"test_loss, test_acc = model.evaluate(X_test, y_test)\n\nprint(f\"Test Accuracy: {test_acc:.2f}\")","metadata":{"id":"f5Mn_4tZcWw8","outputId":"d05ed3b1-20da-4cca-b85b-c877e64cf51a","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:21:16.169458Z","iopub.execute_input":"2025-02-24T17:21:16.169767Z","iopub.status.idle":"2025-02-24T17:21:16.929280Z","shell.execute_reply.started":"2025-02-24T17:21:16.169740Z","shell.execute_reply":"2025-02-24T17:21:16.928034Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Visualize predictions for a few test images","metadata":{"id":"Mo69FRYJc45f"}},{"cell_type":"code","source":"n_images = 10","metadata":{"id":"Sb2flMD_czQB","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:21:16.930501Z","iopub.execute_input":"2025-02-24T17:21:16.930821Z","iopub.status.idle":"2025-02-24T17:21:16.935948Z","shell.execute_reply.started":"2025-02-24T17:21:16.930792Z","shell.execute_reply":"2025-02-24T17:21:16.934366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize= (12,6))\n\nfor i in range(n_images):\n  image = X_test[i].reshape(28,28)\n  label = y_test[i]\n  model_prediction = model.predict(X_test[i].reshape(1,784)).argmax()\n\n  plt.subplot(1, n_images, i+1)\n  plt.imshow(X_test[i].reshape(28,28), cmap='gray')\n  plt.axis('off')\n  plt.title(f\"True: {label}\\nPred: {model_prediction}\")\n\nplt.show()","metadata":{"id":"wlPDSt3PdAD5","outputId":"08a30295-d13a-40ad-9e9d-9b9f9f223cff","trusted":true,"execution":{"iopub.status.busy":"2025-02-24T17:21:16.936979Z","iopub.execute_input":"2025-02-24T17:21:16.937291Z","iopub.status.idle":"2025-02-24T17:21:18.667886Z","shell.execute_reply.started":"2025-02-24T17:21:16.937263Z","shell.execute_reply":"2025-02-24T17:21:18.666037Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<center>\n<pre>\n  _____ _                 _        \n |_   _| |__   __ _ _ __ | | _____ \n   | | | '_ \\ / _` | '_ \\| |/ / __|\n   | | | | | | (_| | | | |   <\\__ \\\n   |_| |_| |_|\\__,_|_| |_|_|\\_\\___/\n       \n</pre>\n</center>","metadata":{}}]}