{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":null,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"76a01d9f-434f-4b42-a3a5-32dfa50a2641","cell_type":"markdown","source":"# Assignment 2: Feed-Forward Neural Network on CIFAR-10\nThis notebook implements the network, backpropagation, and mini-batch SGD manually with NumPy. Run every cell in order.","metadata":{}},{"id":"9fd5a4af-7ed3-4302-a84d-0c740631fe97","cell_type":"code","source":"# Input / setup cell\nimport glob, subprocess, time, warnings\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, confusion_matrix, classification_report\nwarnings.filterwarnings('ignore')\nSEED = 42\nTRAIN_LIMIT, VAL_LIMIT, TEST_LIMIT = 8000, 1000, 1000\nEPOCHS, BATCH_SIZE = 12, 128\nLEARNING_RATES = [0.001, 0.01, 0.1]\nCLASS_NAMES = ['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']\nprint('Setup complete. Kaggle inputs:', glob.glob('/kaggle/input/*'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:24:54.308115Z","iopub.execute_input":"2026-10-06T15:24:54.308443Z","iopub.status.idle":"2026-10-06T15:24:54.315913Z","shell.execute_reply.started":"2026-10-06T15:24:54.308416Z","shell.execute_reply":"2026-10-06T15:24:54.31516Z"}},"outputs":[],"execution_count":null},{"id":"4d5a1caf-d29f-4fdb-b4fc-8c40da96682c","cell_type":"markdown","source":"## Task 1: Load CIFAR-10 directly from the attached Kaggle competition input","metadata":{}},{"id":"c7865aa0-db44-48f8-934e-84ad76cc814c","cell_type":"code","source":"INPUT_ROOT = Path('/kaggle/input')\nlabel_matches = list(INPUT_ROOT.rglob('trainLabels.csv'))\n\nif not label_matches:\n    available = '\\n'.join(str(path) for path in INPUT_ROOT.rglob('*'))\n    raise FileNotFoundError(\n        'Could not find trainLabels.csv in /kaggle/input.\\n'\n        'Attach the CIFAR-10 competition data and restart the session.\\n'\n        f'Currently available inputs:\\n{available or \"(none)\"}'\n    )\n\nDATA_DIR = label_matches[0].parent\nLABELS_FILE = DATA_DIR / 'trainLabels.csv'\nARCHIVE_FILE = DATA_DIR / 'train.7z'\nEXTRACT_DIR = Path('/kaggle/working/cifar10')\nTRAIN_DIR = EXTRACT_DIR / 'train'\n\nif not ARCHIVE_FILE.exists():\n    raise FileNotFoundError(f'Could not find archive: {ARCHIVE_FILE}')\n\nif not TRAIN_DIR.exists():\n    EXTRACT_DIR.mkdir(parents=True, exist_ok=True)\n    subprocess.run(\n        ['7z', 'x', str(ARCHIVE_FILE), f'-o{EXTRACT_DIR}', '-y'],\n        check=True,\n    )\n\nlabels = pd.read_csv(LABELS_FILE)\nclass_to_index = {name: index for index, name in enumerate(CLASS_NAMES)}\nlabels['target'] = labels['label'].map(class_to_index)\n\nif labels['target'].isna().any():\n    unknown = labels.loc[labels['target'].isna(), 'label'].unique()\n    raise ValueError(f'Unknown CIFAR-10 labels found: {unknown}')\n\nlabels['target'] = labels['target'].astype(np.int64)\nsample_count = min(TRAIN_LIMIT + VAL_LIMIT + TEST_LIMIT, len(labels))\n\nif sample_count < len(labels):\n    selected, _ = train_test_split(\n        labels,\n        train_size=sample_count,\n        stratify=labels['target'],\n        random_state=SEED,\n    )\nelse:\n    selected = labels.sample(frac=1, random_state=SEED)\n\nselected = selected.reset_index(drop=True)\nX_selected = np.stack([\n    np.asarray(\n        Image.open(TRAIN_DIR / f'{image_id}.png').convert('RGB'),\n        dtype=np.uint8,\n    )\n    for image_id in selected['id']\n])\ny_selected = selected['target'].to_numpy(dtype=np.int64)\n\nX_train_raw, X_test_raw, y_train_raw, y_test = train_test_split(\n    X_selected,\n    y_selected,\n    test_size=TEST_LIMIT,\n    stratify=y_selected,\n    random_state=SEED,\n)\nX_train_raw, X_val_raw, y_train, y_val = train_test_split(\n    X_train_raw,\n    y_train_raw,\n    test_size=VAL_LIMIT,\n    stratify=y_train_raw,\n    random_state=SEED,\n)\n\nprint(f'Input directory: {DATA_DIR}')\nprint('Loaded from Kaggle input:')\nprint('  Train:', X_train_raw.shape, y_train.shape)\nprint('  Validation:', X_val_raw.shape, y_val.shape)\nprint('  Test:', X_test_raw.shape, y_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:24:57.93168Z","iopub.execute_input":"2026-10-06T15:24:57.932299Z","iopub.status.idle":"2026-10-06T15:25:15.626818Z","shell.execute_reply.started":"2026-10-06T15:24:57.932274Z","shell.execute_reply":"2026-10-06T15:25:15.626025Z"}},"outputs":[],"execution_count":null},{"id":"b0c09e08-6b1d-4712-9494-f73f42544982","cell_type":"markdown","source":"## Task 2: Visualize sample images from the different classes","metadata":{}},{"id":"c856905f-6229-426d-b3c7-d4715cd4bacc","cell_type":"code","source":"fig, axes = plt.subplots(2, 5, figsize=(12, 5))\nfor class_id, ax in enumerate(axes.ravel()):\n    idx = np.flatnonzero(y_train == class_id)[0]\n    ax.imshow(X_train_raw[idx])\n    ax.set_title(CLASS_NAMES[class_id]); ax.axis('off')\nplt.suptitle('CIFAR-10 class samples'); plt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:15.628327Z","iopub.execute_input":"2026-10-06T15:25:15.628637Z","iopub.status.idle":"2026-10-06T15:25:16.183151Z","shell.execute_reply.started":"2026-10-06T15:25:15.628614Z","shell.execute_reply":"2026-10-06T15:25:16.18243Z"}},"outputs":[],"execution_count":null},{"id":"39c891c4-b989-47d1-a1a2-79ca229e74a9","cell_type":"markdown","source":"## Task 3: Normalize pixel values to [0, 1]","metadata":{}},{"id":"56691a31-c11d-4494-b71b-50ab638d2fca","cell_type":"code","source":"X_train_norm = X_train_raw.astype(np.float32) / 255.0\nX_val_norm = X_val_raw.astype(np.float32) / 255.0\nX_test_norm = X_test_raw.astype(np.float32) / 255.0\nprint('Normalized range:', float(X_train_norm.min()), 'to', float(X_train_norm.max()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:16.184092Z","iopub.execute_input":"2026-10-06T15:25:16.184424Z","iopub.status.idle":"2026-10-06T15:25:16.290383Z","shell.execute_reply.started":"2026-10-06T15:25:16.184401Z","shell.execute_reply":"2026-10-06T15:25:16.289476Z"}},"outputs":[],"execution_count":null},{"id":"d84c8bb6-1e55-43e7-b1d9-3b683b454981","cell_type":"markdown","source":"## Task 4: Flatten each image into a one-dimensional feature vector","metadata":{}},{"id":"776b4e1c-f72b-45e3-aab8-a38b617a1c48","cell_type":"code","source":"X_train = X_train_norm.reshape(len(X_train_norm), -1)\nX_val = X_val_norm.reshape(len(X_val_norm), -1)\nX_test = X_test_norm.reshape(len(X_test_norm), -1)\nprint('Flattened shapes:', X_train.shape, X_val.shape, X_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:16.292139Z","iopub.execute_input":"2026-10-06T15:25:16.292423Z","iopub.status.idle":"2026-10-06T15:25:16.298099Z","shell.execute_reply.started":"2026-10-06T15:25:16.292401Z","shell.execute_reply":"2026-10-06T15:25:16.29733Z"}},"outputs":[],"execution_count":null},{"id":"e9cf0354-0963-4514-ab0b-3f33fff72ff6","cell_type":"markdown","source":"## Task 5: Create the MLP architecture (input, two hidden layers, 10-neuron output)","metadata":{}},{"id":"5973e3b0-acac-41b1-9198-c1f416ea4727","cell_type":"code","source":"INPUT_SIZE = X_train.shape[1]\nHIDDEN_1, HIDDEN_2, OUTPUT_SIZE = 64, 32, 10\nLAYER_SIZES = [INPUT_SIZE, HIDDEN_1, HIDDEN_2, OUTPUT_SIZE]\nprint('Architecture:', ' -> '.join(map(str, LAYER_SIZES)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:16.299039Z","iopub.execute_input":"2026-10-06T15:25:16.299675Z","iopub.status.idle":"2026-10-06T15:25:16.314404Z","shell.execute_reply.started":"2026-10-06T15:25:16.299639Z","shell.execute_reply":"2026-10-06T15:25:16.313635Z"}},"outputs":[],"execution_count":null},{"id":"20a37490-4ca6-4d87-837a-8d5deca9838d","cell_type":"markdown","source":"## Task 6: Initialize weights and biases (He initialization for ReLU layers)","metadata":{}},{"id":"4f564b7e-6035-4b79-9555-675022cb36c1","cell_type":"code","source":"def initialize_parameters(layer_sizes, seed=SEED):\n    rng = np.random.default_rng(seed)\n    params = {}\n    for layer in range(1, len(layer_sizes)):\n        fan_in, fan_out = layer_sizes[layer-1], layer_sizes[layer]\n        params[f'W{layer}'] = (rng.standard_normal((fan_in, fan_out)) * np.sqrt(2.0 / fan_in)).astype(np.float32)\n        params[f'b{layer}'] = np.zeros((1, fan_out), dtype=np.float32)\n    return params\ninitial_params = initialize_parameters(LAYER_SIZES)\nfor name, value in initial_params.items(): print(name, value.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:16.315515Z","iopub.execute_input":"2026-10-06T15:25:16.315871Z","iopub.status.idle":"2026-10-06T15:25:16.343485Z","shell.execute_reply.started":"2026-10-06T15:25:16.31584Z","shell.execute_reply":"2026-10-06T15:25:16.342525Z"}},"outputs":[],"execution_count":null},{"id":"81914fdf-df8c-4b0d-9243-0da97712cd84","cell_type":"markdown","source":"## Task 7: Forward propagation, activations, softmax, cross-entropy, backpropagation, and gradients","metadata":{}},{"id":"020affcf-04e3-451a-85d3-5ed6826bf5bd","cell_type":"code","source":"def relu(z): return np.maximum(z, 0)\ndef softmax(z):\n    shifted = z - z.max(axis=1, keepdims=True)\n    exp_z = np.exp(shifted)\n    return exp_z / exp_z.sum(axis=1, keepdims=True)\ndef forward(X, p):\n    z1 = X @ p['W1'] + p['b1']; a1 = relu(z1)\n    z2 = a1 @ p['W2'] + p['b2']; a2 = relu(z2)\n    z3 = a2 @ p['W3'] + p['b3']; probs = softmax(z3)\n    return probs, (X, z1, a1, z2, a2)\ndef cross_entropy(probs, y):\n    return -np.mean(np.log(np.clip(probs[np.arange(len(y)), y], 1e-12, 1.0)))\ndef backward(probs, y, cache, p):\n    X, z1, a1, z2, a2 = cache; m = len(y)\n    dz3 = probs.copy(); dz3[np.arange(m), y] -= 1; dz3 /= m\n    dW3 = a2.T @ dz3; db3 = dz3.sum(axis=0, keepdims=True)\n    dz2 = (dz3 @ p['W3'].T) * (z2 > 0)\n    dW2 = a1.T @ dz2; db2 = dz2.sum(axis=0, keepdims=True)\n    dz1 = (dz2 @ p['W2'].T) * (z1 > 0)\n    dW1 = X.T @ dz1; db1 = dz1.sum(axis=0, keepdims=True)\n    return {'W1': dW1, 'b1': db1, 'W2': dW2, 'b2': db2, 'W3': dW3, 'b3': db3}\ndef evaluate(X, y, p):\n    probs, _ = forward(X, p)\n    return cross_entropy(probs, y), np.mean(probs.argmax(axis=1) == y)\ndemo_probs, demo_cache = forward(X_train[:8], initial_params)\ndemo_grads = backward(demo_probs, y_train[:8], demo_cache, initial_params)\nprint('Initial batch loss:', cross_entropy(demo_probs, y_train[:8]), '| gradient shapes:', {k:v.shape for k,v in demo_grads.items()})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:16.344598Z","iopub.execute_input":"2026-10-06T15:25:16.344903Z","iopub.status.idle":"2026-10-06T15:25:16.366604Z","shell.execute_reply.started":"2026-10-06T15:25:16.344874Z","shell.execute_reply":"2026-10-06T15:25:16.365885Z"}},"outputs":[],"execution_count":null},{"id":"f7aff1a1-65c7-4b79-8332-a3dc7159bc28","cell_type":"markdown","source":"## Task 8: Implement the manual SGD parameter update","metadata":{}},{"id":"c248e465-fefe-4253-acbd-e02d5ada7daf","cell_type":"code","source":"def sgd_update(params, gradients, learning_rate):\n    for name in params:\n        params[name] -= learning_rate * gradients[name]\n    return params\ntest_params = {k:v.copy() for k,v in initial_params.items()}\nold_value = float(test_params['W1'][0, 0])\nsgd_update(test_params, demo_grads, 0.01)\nprint('Example W1[0,0] update:', old_value, '->', float(test_params['W1'][0, 0]))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:16.367842Z","iopub.execute_input":"2026-10-06T15:25:16.368526Z","iopub.status.idle":"2026-10-06T15:25:16.386246Z","shell.execute_reply.started":"2026-10-06T15:25:16.368496Z","shell.execute_reply":"2026-10-06T15:25:16.385561Z"}},"outputs":[],"execution_count":null},{"id":"7f7b88f0-1b6c-4ee1-a224-09b084a3ea5e","cell_type":"markdown","source":"## Task 9: Train with shuffled mini-batches for an appropriate number of epochs","metadata":{}},{"id":"3a222fbe-8e6c-4378-965e-7f6e6764fd04","cell_type":"code","source":"def train_model(learning_rate, epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=True):\n    params = initialize_parameters(LAYER_SIZES)\n    best_params = {k:v.copy() for k,v in params.items()}\n    best_val_acc = -1.0\n    history = {'train_loss':[], 'val_loss':[], 'train_acc':[], 'val_acc':[]}\n    rng = np.random.default_rng(SEED)\n    for epoch in range(1, epochs + 1):\n        order = rng.permutation(len(X_train))\n        for start in range(0, len(order), batch_size):\n            idx = order[start:start + batch_size]\n            probs, cache = forward(X_train[idx], params)\n            grads = backward(probs, y_train[idx], cache, params)\n            sgd_update(params, grads, learning_rate)\n        train_loss, train_acc = evaluate(X_train, y_train, params)\n        val_loss, val_acc = evaluate(X_val, y_val, params)\n        for key, value in zip(history, [train_loss, val_loss, train_acc, val_acc]): history[key].append(value)\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            best_params = {k:v.copy() for k,v in params.items()}\n        if verbose: print(f'lr={learning_rate:g} epoch {epoch:02d}/{epochs} - loss={train_loss:.4f} acc={train_acc:.4f} val_loss={val_loss:.4f} val_acc={val_acc:.4f}')\n    return best_params, history\n\nstart_time = time.time()\nbaseline_model, baseline_history = train_model(0.01)\nprint(f'Baseline training time: {time.time()-start_time:.1f} seconds')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:16.387461Z","iopub.execute_input":"2026-10-06T15:25:16.388117Z","iopub.status.idle":"2026-10-06T15:25:18.335058Z","shell.execute_reply.started":"2026-10-06T15:25:16.38809Z","shell.execute_reply":"2026-10-06T15:25:18.334411Z"}},"outputs":[],"execution_count":null},{"id":"4717a4bf-bc20-4519-84a1-527f894088eb","cell_type":"markdown","source":"## Task 10: Experiment with learning rates 0.001, 0.01, and 0.1","metadata":{}},{"id":"a7fd4eaf-cec1-4d73-b9f9-45dad544e2bf","cell_type":"code","source":"models = {0.01: baseline_model}\nhistories = {0.01: baseline_history}\nfor lr in LEARNING_RATES:\n    if lr not in models:\n        models[lr], histories[lr] = train_model(lr)\nlr_results = pd.DataFrame([{'Learning rate':lr,\n                            'Best validation accuracy':max(histories[lr]['val_acc']),\n                            'Final training accuracy':histories[lr]['train_acc'][-1]}\n                           for lr in LEARNING_RATES]).set_index('Learning rate')\ndisplay(lr_results.style.format('{:.4f}'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:18.337036Z","iopub.execute_input":"2026-10-06T15:25:18.337303Z","iopub.status.idle":"2026-10-06T15:25:22.307969Z","shell.execute_reply.started":"2026-10-06T15:25:18.33728Z","shell.execute_reply":"2026-10-06T15:25:22.307308Z"}},"outputs":[],"execution_count":null},{"id":"5edb5b83-aa2f-450f-8bfc-de53bf699ca9","cell_type":"markdown","source":"## Task 11: Plot training/validation loss and accuracy versus epochs","metadata":{}},{"id":"e2bc8b48-d1eb-4438-8e64-b17167b5c8d5","cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(14, 9))\nplots = [('train_loss','Training loss'), ('val_loss','Validation loss'), ('train_acc','Training accuracy'), ('val_acc','Validation accuracy')]\nfor ax, (metric, title) in zip(axes.ravel(), plots):\n    for lr in LEARNING_RATES:\n        values = histories[lr][metric]\n        ax.plot(range(1, len(values)+1), values, marker='o', markersize=3, label=f'lr={lr}')\n    ax.set_title(title); ax.set_xlabel('Epoch'); ax.set_ylabel(title); ax.grid(alpha=.3); ax.legend()\nplt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:22.309101Z","iopub.execute_input":"2026-10-06T15:25:22.3098Z","iopub.status.idle":"2026-10-06T15:25:22.986883Z","shell.execute_reply.started":"2026-10-06T15:25:22.309767Z","shell.execute_reply":"2026-10-06T15:25:22.985946Z"}},"outputs":[],"execution_count":null},{"id":"e99b6de5-f5bd-4099-8486-3d005d37fed4","cell_type":"markdown","source":"## Task 12: Test accuracy, confusion matrix, and class-wise accuracy","metadata":{}},{"id":"e7c7a2e3-d818-40af-864e-4bfd2e7425a2","cell_type":"code","source":"best_lr = float(lr_results['Best validation accuracy'].idxmax())\nbest_model = models[best_lr]\ntest_probs, _ = forward(X_test, best_model)\ntest_pred = test_probs.argmax(axis=1)\ntest_accuracy = accuracy_score(y_test, test_pred)\ncm = confusion_matrix(y_test, test_pred, labels=np.arange(10))\nclass_accuracy = np.divide(np.diag(cm), cm.sum(axis=1), out=np.zeros(10, dtype=float), where=cm.sum(axis=1)!=0)\nprint(f'Best learning rate: {best_lr:g}')\nprint(f'Test classification accuracy: {test_accuracy:.4f}')\ndisplay(pd.DataFrame({'Class': CLASS_NAMES, 'Class-wise accuracy': class_accuracy}).style.format({'Class-wise accuracy':'{:.4f}'}))\nplt.figure(figsize=(10, 8))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES)\nplt.title(f'Test confusion matrix (learning rate = {best_lr:g})')\nplt.xlabel('Predicted label'); plt.ylabel('True label'); plt.tight_layout(); plt.show()\nprint(classification_report(y_test, test_pred, target_names=CLASS_NAMES, zero_division=0))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-06T15:25:22.987946Z","iopub.execute_input":"2026-10-06T15:25:22.988257Z","iopub.status.idle":"2026-10-06T15:25:23.460351Z","shell.execute_reply.started":"2026-10-06T15:25:22.988225Z","shell.execute_reply":"2026-10-06T15:25:23.459556Z"}},"outputs":[],"execution_count":null},{"id":"0a76c6e3-474f-488a-a3f0-5f2fc873d907","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"0044e0c1-1841-4df7-b17c-1fd0c87d3401","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}