{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":16017,"databundleVersionId":708189,"isSourceIdPinned":false}],"dockerImageVersionId":31329,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"e4b8a586-50e1-4ba2-a085-4548bfbad410","cell_type":"markdown","source":"# Kannada-MNIST: Handwritten Digit Recognition\n### End-to-End Project: Classical ML vs Neural Networks\n\n**Dataset:** [Kaggle – Kannada MNIST](https://www.kaggle.com/competitions/Kannada-MNIST)  \n**Task:** Multi-class classification of handwritten Kannada digits (0–9)  \n**Framework:** scikit-learn (classical ML) + PyTorch (neural networks)\n\n---\n## Table of Contents\n1. Setup & Imports\n2. Data Loading & Exploration\n3. Preprocessing\n4. Classical ML Baseline (KNN, SVM, Random Forest)\n5. Neural Network – MLP (PyTorch)\n6. Neural Network – CNN (PyTorch)\n7. Model Evaluation & Comparison\n8. Error Analysis\n9. Kaggle Submission\n10. Summary Report","metadata":{}},{"id":"db1de7b6-006b-421d-baa7-302c6d93df84","cell_type":"markdown","source":"## 1. Setup & Imports","metadata":{}},{"id":"2bcf9d52-dbba-4240-9ab7-61218e45fbec","cell_type":"code","source":"# !pip install torch torchvision scikit-learn pandas matplotlib seaborn\n\nimport os, time, warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import defaultdict\n\n# ── scikit-learn ─────────────────────────────────────────────────────────────\nfrom sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import (\n    accuracy_score, classification_report,\n    confusion_matrix, ConfusionMatrixDisplay\n)\n\n# ── PyTorch ──────────────────────────────────────────────────────────────────\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset, random_split\nfrom torchvision import transforms\n\n# ── Reproducibility ──────────────────────────────────────────────────────────\nSEED = 42\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Using device: {DEVICE}')\nprint(f'PyTorch version: {torch.__version__}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:23:48.949639Z","iopub.execute_input":"2026-04-08T09:23:48.949887Z","iopub.status.idle":"2026-04-08T09:24:06.416182Z","shell.execute_reply.started":"2026-04-08T09:23:48.949857Z","shell.execute_reply":"2026-04-08T09:24:06.415495Z"}},"outputs":[],"execution_count":null},{"id":"9ed59ea5-5860-4161-8ae7-899eec189df1","cell_type":"markdown","source":"## 2. Data Loading & Exploration","metadata":{}},{"id":"b3f7b422-6195-46c8-9afa-92aa677b50f3","cell_type":"code","source":"# ── Load data ────────────────────────────────────────────────────────────────\nDATA_DIR = '/kaggle/input/competitions/Kannada-MNIST'\n\ntrain_df  = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))\ntest_df   = pd.read_csv(os.path.join(DATA_DIR, 'test.csv'))\ndig_df    = pd.read_csv(os.path.join(DATA_DIR, 'Dig-MNIST.csv'))\n\nprint('Train shape :', train_df.shape)\nprint('Test shape  :', test_df.shape)\nprint('Dig-MNIST   :', dig_df.shape)\ntrain_df.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:06.417874Z","iopub.execute_input":"2026-04-08T09:24:06.418311Z","iopub.status.idle":"2026-04-08T09:24:11.636794Z","shell.execute_reply.started":"2026-04-08T09:24:06.418255Z","shell.execute_reply":"2026-04-08T09:24:11.636077Z"}},"outputs":[],"execution_count":null},{"id":"b9a1e248-de3b-49c2-b60f-565e936e1fc3","cell_type":"code","source":"# ── Separate labels and features ─────────────────────────────────────────────\nX_all = train_df.drop('label', axis=1).values   # (60000, 784)\ny_all = train_df['label'].values                 # (60000,)\n\nX_test_kaggle = test_df.drop('id', axis=1).values\n\nX_dig = dig_df.drop('label', axis=1).values\ny_dig = dig_df['label'].values\n\nprint('X_all:', X_all.shape, '| unique labels:', np.unique(y_all))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:11.637883Z","iopub.execute_input":"2026-04-08T09:24:11.638466Z","iopub.status.idle":"2026-04-08T09:24:11.784749Z","shell.execute_reply.started":"2026-04-08T09:24:11.638438Z","shell.execute_reply":"2026-04-08T09:24:11.784083Z"}},"outputs":[],"execution_count":null},{"id":"eeb31f55-c1d0-4432-877c-c6690c87aeb2","cell_type":"code","source":"# ── 2.1 Class distribution ───────────────────────────────────────────────────\nfig, axes = plt.subplots(1, 2, figsize=(14, 4))\n\ncounts = pd.Series(y_all).value_counts().sort_index()\naxes[0].bar(counts.index, counts.values, color='steelblue', edgecolor='white')\naxes[0].set_title('Train – Class Distribution', fontsize=13)\naxes[0].set_xlabel('Digit'); axes[0].set_ylabel('Count')\naxes[0].set_xticks(range(10))\nfor i, v in enumerate(counts.values):\n    axes[0].text(i, v + 30, str(v), ha='center', fontsize=9)\n\ncounts_dig = pd.Series(y_dig).value_counts().sort_index()\naxes[1].bar(counts_dig.index, counts_dig.values, color='coral', edgecolor='white')\naxes[1].set_title('Dig-MNIST – Class Distribution', fontsize=13)\naxes[1].set_xlabel('Digit'); axes[1].set_ylabel('Count')\naxes[1].set_xticks(range(10))\n\nplt.tight_layout()\nplt.savefig('class_distribution.png', dpi=120, bbox_inches='tight')\nplt.show()\nprint('Classes are well-balanced:', counts.values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:11.785676Z","iopub.execute_input":"2026-04-08T09:24:11.785939Z","iopub.status.idle":"2026-04-08T09:24:12.419815Z","shell.execute_reply.started":"2026-04-08T09:24:11.785914Z","shell.execute_reply":"2026-04-08T09:24:12.419017Z"}},"outputs":[],"execution_count":null},{"id":"4365c8c6-c0af-4a0f-a5a0-4eb109ef3bf3","cell_type":"code","source":"# ── 2.2 Sample images per class ──────────────────────────────────────────────\nfig, axes = plt.subplots(10, 8, figsize=(14, 18))\nfig.suptitle('Sample Images per Kannada Digit (0–9)', fontsize=16, y=1.01)\n\nfor digit in range(10):\n    idxs = np.where(y_all == digit)[0][:8]\n    for col, idx in enumerate(idxs):\n        ax = axes[digit][col]\n        ax.imshow(X_all[idx].reshape(28, 28), cmap='gray_r')\n        ax.axis('off')\n        if col == 0:\n            ax.set_ylabel(f'Digit {digit}', fontsize=11, rotation=0, labelpad=40, va='center')\n\nplt.tight_layout()\nplt.savefig('sample_images.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:12.420718Z","iopub.execute_input":"2026-04-08T09:24:12.420938Z","iopub.status.idle":"2026-04-08T09:24:14.465127Z","shell.execute_reply.started":"2026-04-08T09:24:12.420915Z","shell.execute_reply":"2026-04-08T09:24:14.464508Z"}},"outputs":[],"execution_count":null},{"id":"9994a3a5-c32d-49c3-b40c-0c8f4ab5d9ad","cell_type":"code","source":"# ── 2.3 Pixel statistics ──────────────────────────────────────────────────────\nprint(f'Pixel value range  : [{X_all.min()}, {X_all.max()}]')\nprint(f'Mean pixel value   : {X_all.mean():.2f}')\nprint(f'Std  pixel value   : {X_all.std():.2f}')\n\n# Mean image per class\nfig, axes = plt.subplots(1, 10, figsize=(18, 2))\nfor digit in range(10):\n    mean_img = X_all[y_all == digit].mean(axis=0).reshape(28, 28)\n    axes[digit].imshow(mean_img, cmap='gray_r')\n    axes[digit].set_title(f'{digit}'); axes[digit].axis('off')\nfig.suptitle('Mean Image per Digit Class', fontsize=13)\nplt.tight_layout()\nplt.savefig('mean_images.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:14.465986Z","iopub.execute_input":"2026-04-08T09:24:14.466261Z","iopub.status.idle":"2026-04-08T09:24:15.989009Z","shell.execute_reply.started":"2026-04-08T09:24:14.466236Z","shell.execute_reply":"2026-04-08T09:24:15.988416Z"}},"outputs":[],"execution_count":null},{"id":"0970ce25-4801-4208-b6cd-cf0657879c66","cell_type":"markdown","source":"## 3. Preprocessing","metadata":{}},{"id":"02f91436-a548-49d2-99b0-a737b515d6fb","cell_type":"code","source":"# ── 3.1 Normalize to [0, 1] ──────────────────────────────────────────────────\nX_norm = X_all.astype(np.float32) / 255.0\nX_dig_norm = X_dig.astype(np.float32) / 255.0\nX_test_norm = X_test_kaggle.astype(np.float32) / 255.0\n\n# ── 3.2 Train / validation split (80/20) ────────────────────────────────────\nX_train, X_val, y_train, y_val = train_test_split(\n    X_norm, y_all, test_size=0.2, random_state=SEED, stratify=y_all\n)\nprint(f'Train: {X_train.shape} | Val: {X_val.shape}')\n\n# ── 3.3 Reshape to images (for CNN) ─────────────────────────────────────────\n# Shape: (N, 1, 28, 28)\nX_train_img = X_train.reshape(-1, 1, 28, 28)\nX_val_img   = X_val.reshape(-1, 1, 28, 28)\nX_dig_img   = X_dig_norm.reshape(-1, 1, 28, 28)\nX_test_img  = X_test_norm.reshape(-1, 1, 28, 28)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:15.991171Z","iopub.execute_input":"2026-04-08T09:24:15.991482Z","iopub.status.idle":"2026-04-08T09:24:16.698988Z","shell.execute_reply.started":"2026-04-08T09:24:15.991457Z","shell.execute_reply":"2026-04-08T09:24:16.698354Z"}},"outputs":[],"execution_count":null},{"id":"e7127095-bdd1-4253-9133-65b6c52ed553","cell_type":"code","source":"# ── 3.4 PCA for classical ML (speed up KNN/SVM) ───────────────────────────────\n# Keep 95% of variance\npca = PCA(n_components=0.95, random_state=SEED)\nX_train_pca = pca.fit_transform(X_train)\nX_val_pca   = pca.transform(X_val)\nX_dig_pca   = pca.transform(X_dig_norm)\nX_test_pca  = pca.transform(X_test_norm)\n\nprint(f'PCA: {X_train.shape[1]} → {X_train_pca.shape[1]} features')\nprint(f'Explained variance: {pca.explained_variance_ratio_.sum():.3f}')\n\n# Optional: visualize explained variance\ncumvar = np.cumsum(pca.explained_variance_ratio_)\nplt.figure(figsize=(8, 3))\nplt.plot(cumvar, color='steelblue')\nplt.axhline(0.95, color='red', linestyle='--', label='95% threshold')\nplt.xlabel('Number of PCA components')\nplt.ylabel('Cumulative explained variance')\nplt.title('PCA – Cumulative Explained Variance')\nplt.legend(); plt.tight_layout()\nplt.savefig('pca_variance.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:16.699915Z","iopub.execute_input":"2026-04-08T09:24:16.700147Z","iopub.status.idle":"2026-04-08T09:24:17.560025Z","shell.execute_reply.started":"2026-04-08T09:24:16.700124Z","shell.execute_reply":"2026-04-08T09:24:17.559317Z"}},"outputs":[],"execution_count":null},{"id":"34dca8b4-a239-4afe-b09c-756c8eadcb67","cell_type":"markdown","source":"## 4. Classical ML Baseline","metadata":{}},{"id":"4db6a5a4-2fc3-4eff-ac52-752596b38c0b","cell_type":"markdown","source":"### 4.1 K-Nearest Neighbors","metadata":{}},{"id":"56db20f4-c458-4cc7-a171-33e14e3431bf","cell_type":"code","source":"# ── KNN with cross-validated hyperparameter search ───────────────────────────\n# Using a 10k subset for speed during grid search\nN_SUB = 10000\nrng = np.random.RandomState(SEED)\nsub_idx = rng.choice(len(X_train_pca), N_SUB, replace=False)\nXs, ys = X_train_pca[sub_idx], y_train[sub_idx]\n\nparam_grid_knn = {'n_neighbors': [3, 5, 7, 11]}\nknn_gs = GridSearchCV(\n    KNeighborsClassifier(metric='euclidean', n_jobs=-1),\n    param_grid_knn, cv=3, scoring='accuracy', n_jobs=-1, verbose=1\n)\nknn_gs.fit(Xs, ys)\nprint('Best KNN params:', knn_gs.best_params_)\nprint('Best CV accuracy:', f\"{knn_gs.best_score_:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:17.561042Z","iopub.execute_input":"2026-04-08T09:24:17.561469Z","iopub.status.idle":"2026-04-08T09:24:21.681795Z","shell.execute_reply.started":"2026-04-08T09:24:17.561432Z","shell.execute_reply":"2026-04-08T09:24:21.680901Z"}},"outputs":[],"execution_count":null},{"id":"9161fb49-4bdd-421f-a3d8-7662f7fc5677","cell_type":"code","source":"# ── Train best KNN on full training set ──────────────────────────────────────\nbest_k = knn_gs.best_params_['n_neighbors']\nknn = KNeighborsClassifier(n_neighbors=best_k, metric='euclidean', n_jobs=-1)\nknn.fit(X_train_pca, y_train)\n\nknn_val_preds  = knn.predict(X_val_pca)\nknn_val_acc    = accuracy_score(y_val, knn_val_preds)\nknn_dig_preds  = knn.predict(X_dig_pca)\nknn_dig_acc    = accuracy_score(y_dig, knn_dig_preds)\n\nprint(f'KNN (k={best_k}) Validation Accuracy : {knn_val_acc:.4f}')\nprint(f'KNN (k={best_k}) Dig-MNIST  Accuracy : {knn_dig_acc:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:21.682864Z","iopub.execute_input":"2026-04-08T09:24:21.683248Z","iopub.status.idle":"2026-04-08T09:24:29.895340Z","shell.execute_reply.started":"2026-04-08T09:24:21.683222Z","shell.execute_reply":"2026-04-08T09:24:29.894455Z"}},"outputs":[],"execution_count":null},{"id":"7b140818-766c-4736-806c-d59ae2090859","cell_type":"markdown","source":"### 4.2 Support Vector Machine","metadata":{}},{"id":"612758d2-46f9-464d-bc60-695a8a93226b","cell_type":"code","source":"# ── SVM with RBF kernel ───────────────────────────────────────────────────────\nparam_grid_svm = {'C': [1, 10], 'gamma': ['scale', 'auto']}\nsvm_gs = GridSearchCV(\n    SVC(kernel='rbf', random_state=SEED),\n    param_grid_svm, cv=3, scoring='accuracy', n_jobs=-1, verbose=1\n)\nsvm_gs.fit(Xs, ys)\nprint('Best SVM params:', svm_gs.best_params_)\nprint('Best CV accuracy:', f\"{svm_gs.best_score_:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:29.896515Z","iopub.execute_input":"2026-04-08T09:24:29.896847Z","iopub.status.idle":"2026-04-08T09:24:59.414032Z","shell.execute_reply.started":"2026-04-08T09:24:29.896820Z","shell.execute_reply":"2026-04-08T09:24:59.413330Z"}},"outputs":[],"execution_count":null},{"id":"e54c567f-647e-4953-9d63-1ed0d26215fa","cell_type":"code","source":"# ── Train best SVM on full training set ──────────────────────────────────────\nsvm = SVC(**svm_gs.best_params_, kernel='rbf', random_state=SEED)\nsvm.fit(X_train_pca, y_train)\n\nsvm_val_preds  = svm.predict(X_val_pca)\nsvm_val_acc    = accuracy_score(y_val, svm_val_preds)\nsvm_dig_preds  = svm.predict(X_dig_pca)\nsvm_dig_acc    = accuracy_score(y_dig, svm_dig_preds)\n\nprint(f'SVM Validation Accuracy : {svm_val_acc:.4f}')\nprint(f'SVM Dig-MNIST  Accuracy : {svm_dig_acc:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:24:59.415022Z","iopub.execute_input":"2026-04-08T09:24:59.415326Z","iopub.status.idle":"2026-04-08T09:26:53.669039Z","shell.execute_reply.started":"2026-04-08T09:24:59.415267Z","shell.execute_reply":"2026-04-08T09:26:53.668365Z"}},"outputs":[],"execution_count":null},{"id":"ce472f2d-3b32-41a6-8eb8-a5eac4d2f256","cell_type":"markdown","source":"### 4.3 Random Forest","metadata":{}},{"id":"2495f1e0-e2ad-449a-96e7-f9d536c78777","cell_type":"code","source":"# ── Random Forest with cross-validated hyperparameter search ─────────────────\nparam_grid_rf = {'n_estimators': [100, 200], 'max_depth': [None, 30]}\nrf_gs = GridSearchCV(\n    RandomForestClassifier(random_state=SEED, n_jobs=-1),\n    param_grid_rf, cv=3, scoring='accuracy', n_jobs=-1, verbose=1\n)\nrf_gs.fit(Xs, ys)\nprint('Best RF params:', rf_gs.best_params_)\nprint('Best CV accuracy:', f\"{rf_gs.best_score_:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:26:53.670233Z","iopub.execute_input":"2026-04-08T09:26:53.670593Z","iopub.status.idle":"2026-04-08T09:28:38.142090Z","shell.execute_reply.started":"2026-04-08T09:26:53.670553Z","shell.execute_reply":"2026-04-08T09:28:38.141352Z"}},"outputs":[],"execution_count":null},{"id":"f43451fc-779e-46d8-8433-173c57ef2358","cell_type":"code","source":"# ── Train best RF on full training set ───────────────────────────────────────\nrf = RandomForestClassifier(**rf_gs.best_params_, random_state=SEED, n_jobs=-1)\nrf.fit(X_train_pca, y_train)\n\nrf_val_preds  = rf.predict(X_val_pca)\nrf_val_acc    = accuracy_score(y_val, rf_val_preds)\nrf_dig_preds  = rf.predict(X_dig_pca)\nrf_dig_acc    = accuracy_score(y_dig, rf_dig_preds)\n\nprint(f'Random Forest Validation Accuracy : {rf_val_acc:.4f}')\nprint(f'Random Forest Dig-MNIST  Accuracy : {rf_dig_acc:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:28:38.143237Z","iopub.execute_input":"2026-04-08T09:28:38.143594Z","iopub.status.idle":"2026-04-08T09:30:12.965139Z","shell.execute_reply.started":"2026-04-08T09:28:38.143556Z","shell.execute_reply":"2026-04-08T09:30:12.964506Z"}},"outputs":[],"execution_count":null},{"id":"6b2b05b1-07c4-4531-887d-f3d93c260996","cell_type":"markdown","source":"## 5. Neural Network – MLP (PyTorch)","metadata":{}},{"id":"62638b81-7c76-4957-86d0-50dc6c77d2a5","cell_type":"code","source":"# ── PyTorch Dataset helpers ───────────────────────────────────────────────────\ndef make_loader(X, y, batch_size=256, shuffle=True):\n    Xt = torch.tensor(X, dtype=torch.float32)\n    yt = torch.tensor(y, dtype=torch.long)\n    return DataLoader(TensorDataset(Xt, yt), batch_size=batch_size, shuffle=shuffle)\n\ntrain_loader = make_loader(X_train, y_train, batch_size=256)\nval_loader   = make_loader(X_val,   y_val,   shuffle=False)\ndig_loader   = make_loader(X_dig_norm, y_dig, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:30:12.966561Z","iopub.execute_input":"2026-04-08T09:30:12.966877Z","iopub.status.idle":"2026-04-08T09:30:13.131181Z","shell.execute_reply.started":"2026-04-08T09:30:12.966841Z","shell.execute_reply":"2026-04-08T09:30:13.130513Z"}},"outputs":[],"execution_count":null},{"id":"2b5228f4-284b-41f8-8ea8-47554e4d0d8f","cell_type":"code","source":"# ── MLP Architecture ──────────────────────────────────────────────────────────\nclass MLP(nn.Module):\n    def __init__(self, hidden_sizes=(512, 256, 128), dropout=0.3):\n        super().__init__()\n        layers = []\n        in_size = 784\n        for h in hidden_sizes:\n            layers += [\n                nn.Linear(in_size, h),\n                nn.BatchNorm1d(h),\n                nn.ReLU(),\n                nn.Dropout(dropout)\n            ]\n            in_size = h\n        layers.append(nn.Linear(in_size, 10))\n        self.net = nn.Sequential(*layers)\n\n    def forward(self, x):\n        return self.net(x)\n\n# Test forward pass\nmlp = MLP().to(DEVICE)\ndummy = torch.zeros(4, 784).to(DEVICE)\nprint('MLP output shape:', mlp(dummy).shape)\nprint(mlp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:30:13.132067Z","iopub.execute_input":"2026-04-08T09:30:13.132298Z","iopub.status.idle":"2026-04-08T09:30:14.028336Z","shell.execute_reply.started":"2026-04-08T09:30:13.132259Z","shell.execute_reply":"2026-04-08T09:30:14.027688Z"}},"outputs":[],"execution_count":null},{"id":"3b0dc7f3-019f-4f53-baa9-f1046d702e21","cell_type":"code","source":"# ── Training loop ─────────────────────────────────────────────────────────────\ndef train_model(model, train_loader, val_loader, epochs=30,\n                lr=1e-3, patience=5, scheduler_step=10):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-4)\n    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=scheduler_step, gamma=0.5)\n\n    history = defaultdict(list)\n    best_val_acc, best_state, patience_cnt = 0.0, None, 0\n\n    for epoch in range(1, epochs + 1):\n        # ── Train ──\n        model.train()\n        train_loss, correct, total = 0.0, 0, 0\n        for X_batch, y_batch in train_loader:\n            X_batch, y_batch = X_batch.to(DEVICE), y_batch.to(DEVICE)\n            optimizer.zero_grad()\n            out = model(X_batch)\n            loss = criterion(out, y_batch)\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item() * len(y_batch)\n            correct    += (out.argmax(1) == y_batch).sum().item()\n            total      += len(y_batch)\n        scheduler.step()\n\n        # ── Validate ──\n        model.eval()\n        val_loss, val_correct, val_total = 0.0, 0, 0\n        with torch.no_grad():\n            for X_batch, y_batch in val_loader:\n                X_batch, y_batch = X_batch.to(DEVICE), y_batch.to(DEVICE)\n                out = model(X_batch)\n                loss = criterion(out, y_batch)\n                val_loss    += loss.item() * len(y_batch)\n                val_correct += (out.argmax(1) == y_batch).sum().item()\n                val_total   += len(y_batch)\n\n        tr_acc  = correct / total\n        val_acc = val_correct / val_total\n        history['train_loss'].append(train_loss / total)\n        history['val_loss'].append(val_loss / val_total)\n        history['train_acc'].append(tr_acc)\n        history['val_acc'].append(val_acc)\n\n        if epoch % 5 == 0:\n            print(f'Epoch {epoch:3d} | '\n                  f'Train Acc: {tr_acc:.4f} | Val Acc: {val_acc:.4f} | '\n                  f'Val Loss: {val_loss/val_total:.4f}')\n\n        # ── Early stopping ──\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            best_state   = {k: v.clone() for k, v in model.state_dict().items()}\n            patience_cnt = 0\n        else:\n            patience_cnt += 1\n            if patience_cnt >= patience:\n                print(f'Early stopping at epoch {epoch}')\n                break\n\n    model.load_state_dict(best_state)\n    print(f'Best validation accuracy: {best_val_acc:.4f}')\n    return history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:30:14.029231Z","iopub.execute_input":"2026-04-08T09:30:14.029581Z","iopub.status.idle":"2026-04-08T09:30:14.039668Z","shell.execute_reply.started":"2026-04-08T09:30:14.029556Z","shell.execute_reply":"2026-04-08T09:30:14.039063Z"}},"outputs":[],"execution_count":null},{"id":"171f2691-64e1-4d1c-b7dd-6d81872de3d5","cell_type":"code","source":"# ── Plot training curves ──────────────────────────────────────────────────────\ndef plot_history(history, title='Training Curves', filename=None):\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 4))\n    epochs = range(1, len(history['train_loss']) + 1)\n\n    ax1.plot(epochs, history['train_loss'], label='Train Loss', color='steelblue')\n    ax1.plot(epochs, history['val_loss'],   label='Val Loss',   color='coral')\n    ax1.set_title('Loss'); ax1.set_xlabel('Epoch'); ax1.legend()\n\n    ax2.plot(epochs, history['train_acc'], label='Train Acc', color='steelblue')\n    ax2.plot(epochs, history['val_acc'],   label='Val Acc',   color='coral')\n    ax2.set_title('Accuracy'); ax2.set_xlabel('Epoch'); ax2.legend()\n\n    fig.suptitle(title, fontsize=13)\n    plt.tight_layout()\n    if filename:\n        plt.savefig(filename, dpi=120, bbox_inches='tight')\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:30:14.040711Z","iopub.execute_input":"2026-04-08T09:30:14.040947Z","iopub.status.idle":"2026-04-08T09:30:14.063651Z","shell.execute_reply.started":"2026-04-08T09:30:14.040922Z","shell.execute_reply":"2026-04-08T09:30:14.063100Z"}},"outputs":[],"execution_count":null},{"id":"c39ae423-4c25-40c2-be99-b7b43df22151","cell_type":"code","source":"# ── 5.1 Train MLP (default config) ───────────────────────────────────────────\nprint('=== MLP – Default config (dropout=0.3, lr=1e-3) ===')\nmlp = MLP(hidden_sizes=(512, 256, 128), dropout=0.3).to(DEVICE)\nhistory_mlp = train_model(mlp, train_loader, val_loader, epochs=40, lr=1e-3, patience=7)\nplot_history(history_mlp, title='MLP Training Curves', filename='mlp_curves.png')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:30:14.064432Z","iopub.execute_input":"2026-04-08T09:30:14.064671Z","iopub.status.idle":"2026-04-08T09:30:41.821833Z","shell.execute_reply.started":"2026-04-08T09:30:14.064648Z","shell.execute_reply":"2026-04-08T09:30:41.821220Z"}},"outputs":[],"execution_count":null},{"id":"da5405c3-9957-4d12-b75b-9a9a83c673bf","cell_type":"code","source":"# ── 5.2 Hyperparameter experiment: dropout ────────────────────────────────────\nprint('\\n=== MLP – High dropout (0.5) ===')\nmlp_high_drop = MLP(hidden_sizes=(512, 256, 128), dropout=0.5).to(DEVICE)\nhistory_mlp_hd = train_model(mlp_high_drop, train_loader, val_loader,\n                              epochs=40, lr=1e-3, patience=7)\n\n# Compare val accuracy\nfig, ax = plt.subplots(figsize=(8, 4))\nax.plot(history_mlp['val_acc'],    label='Dropout=0.3')\nax.plot(history_mlp_hd['val_acc'], label='Dropout=0.5')\nax.set_title('MLP – Effect of Dropout on Validation Accuracy')\nax.set_xlabel('Epoch'); ax.set_ylabel('Accuracy'); ax.legend()\nplt.tight_layout()\nplt.savefig('mlp_dropout_compare.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:30:41.822721Z","iopub.execute_input":"2026-04-08T09:30:41.822973Z","iopub.status.idle":"2026-04-08T09:31:19.906998Z","shell.execute_reply.started":"2026-04-08T09:30:41.822950Z","shell.execute_reply":"2026-04-08T09:31:19.906367Z"}},"outputs":[],"execution_count":null},{"id":"cd6b814a-225d-4fa8-8bfc-bba11a5b9e11","cell_type":"code","source":"# ── 5.3 Hyperparameter experiment: learning rate ──────────────────────────────\nprint('\\n=== MLP – Lower LR (1e-4) ===')\nmlp_low_lr = MLP(hidden_sizes=(512, 256, 128), dropout=0.3).to(DEVICE)\nhistory_mlp_lr = train_model(mlp_low_lr, train_loader, val_loader,\n                              epochs=40, lr=1e-4, patience=7)\n\nfig, ax = plt.subplots(figsize=(8, 4))\nax.plot(history_mlp['val_acc'],    label='LR=1e-3')\nax.plot(history_mlp_lr['val_acc'], label='LR=1e-4')\nax.set_title('MLP – Effect of Learning Rate on Validation Accuracy')\nax.set_xlabel('Epoch'); ax.set_ylabel('Accuracy'); ax.legend()\nplt.tight_layout()\nplt.savefig('mlp_lr_compare.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:31:19.907983Z","iopub.execute_input":"2026-04-08T09:31:19.908219Z","iopub.status.idle":"2026-04-08T09:31:51.095498Z","shell.execute_reply.started":"2026-04-08T09:31:19.908196Z","shell.execute_reply":"2026-04-08T09:31:51.094918Z"}},"outputs":[],"execution_count":null},{"id":"7ebd2c08-0b53-4563-9469-9225f3e5c68d","cell_type":"code","source":"# ── 5.4 Evaluate best MLP ─────────────────────────────────────────────────────\ndef evaluate_model(model, loader, is_cnn=False):\n    model.eval()\n    all_preds, all_true = [], []\n    with torch.no_grad():\n        for X_batch, y_batch in loader:\n            X_batch = X_batch.to(DEVICE)\n            if not is_cnn:\n                X_batch = X_batch.view(X_batch.size(0), -1)\n            out = model(X_batch)\n            all_preds.extend(out.argmax(1).cpu().numpy())\n            all_true.extend(y_batch.numpy())\n    return np.array(all_preds), np.array(all_true)\n\nmlp_val_preds, _   = evaluate_model(mlp, val_loader)\nmlp_val_acc        = accuracy_score(y_val, mlp_val_preds)\nmlp_dig_preds, _   = evaluate_model(mlp, dig_loader)\nmlp_dig_acc        = accuracy_score(y_dig, mlp_dig_preds)\n\nprint(f'MLP Validation Accuracy : {mlp_val_acc:.4f}')\nprint(f'MLP Dig-MNIST  Accuracy : {mlp_dig_acc:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:31:51.098654Z","iopub.execute_input":"2026-04-08T09:31:51.099058Z","iopub.status.idle":"2026-04-08T09:31:51.382597Z","shell.execute_reply.started":"2026-04-08T09:31:51.099034Z","shell.execute_reply":"2026-04-08T09:31:51.381808Z"}},"outputs":[],"execution_count":null},{"id":"034988b0-5cbc-4bbd-882c-7278ea3a02b3","cell_type":"markdown","source":"## 6. Neural Network – CNN (PyTorch)","metadata":{}},{"id":"00eb18ad-92d2-4e7d-b224-ba72dc8ea045","cell_type":"code","source":"# ── CNN Architecture ──────────────────────────────────────────────────────────\nclass CNN(nn.Module):\n    def __init__(self, dropout=0.3):\n        super().__init__()\n        # Block 1: 1→32 channels\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(1, 32, kernel_size=3, padding=1),\n            nn.BatchNorm2d(32), nn.ReLU(),\n            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n            nn.BatchNorm2d(32), nn.ReLU(),\n            nn.MaxPool2d(2, 2),           # → 14×14\n            nn.Dropout2d(dropout / 2)\n        )\n        # Block 2: 32→64 channels\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(32, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64), nn.ReLU(),\n            nn.Conv2d(64, 64, kernel_size=3, padding=1),\n            nn.BatchNorm2d(64), nn.ReLU(),\n            nn.MaxPool2d(2, 2),           # → 7×7\n            nn.Dropout2d(dropout / 2)\n        )\n        # Block 3: 64→128 channels\n        self.conv3 = nn.Sequential(\n            nn.Conv2d(64, 128, kernel_size=3, padding=1),\n            nn.BatchNorm2d(128), nn.ReLU(),\n            nn.MaxPool2d(2, 2),           # → 3×3\n            nn.Dropout2d(dropout / 2)\n        )\n        # Classifier head\n        self.classifier = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(128 * 3 * 3, 256),\n            nn.BatchNorm1d(256), nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(256, 10)\n        )\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        return self.classifier(x)\n\n# Test\ncnn = CNN().to(DEVICE)\ndummy = torch.zeros(4, 1, 28, 28).to(DEVICE)\nprint('CNN output shape:', cnn(dummy).shape)\ntotal_params = sum(p.numel() for p in cnn.parameters() if p.requires_grad)\nprint(f'Trainable params: {total_params:,}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:31:51.383618Z","iopub.execute_input":"2026-04-08T09:31:51.383903Z","iopub.status.idle":"2026-04-08T09:31:52.433128Z","shell.execute_reply.started":"2026-04-08T09:31:51.383875Z","shell.execute_reply":"2026-04-08T09:31:52.432345Z"}},"outputs":[],"execution_count":null},{"id":"dec0d071-f54c-4293-bdd9-fdfbdaa959da","cell_type":"code","source":"# ── CNN DataLoaders (with optional augmentation) ──────────────────────────────\n# Data augmentation (applied only to training set)\ndef make_augmented_loader(X, y, batch_size=256, augment=False):\n    Xt = torch.tensor(X, dtype=torch.float32)\n    yt = torch.tensor(y, dtype=torch.long)\n    dataset = TensorDataset(Xt, yt)\n\n    if augment:\n        # We'll apply transforms during training manually\n        pass   # handled in training loop via transform below\n\n    return DataLoader(dataset, batch_size=batch_size, shuffle=(y is not None))\n\n# Augmentation transform (applied on-the-fly)\naug_transform = transforms.Compose([\n    transforms.RandomAffine(degrees=10, translate=(0.1, 0.1), scale=(0.9, 1.1)),\n])\n\ncnn_train_loader = make_loader(X_train_img.reshape(len(X_train_img), -1),\n                                y_train, batch_size=128)\ncnn_val_loader   = make_loader(X_val_img.reshape(len(X_val_img), -1),\n                                y_val, batch_size=128, shuffle=False)\ncnn_dig_loader   = make_loader(X_dig_img.reshape(len(X_dig_img), -1),\n                                y_dig, batch_size=128, shuffle=False)\n\n# Convenience wrapper: CNN needs (N,1,28,28), MLP flattens internally.\n# We store flat tensors and reshape in the CNN training loop.\nprint('CNN data loaders ready.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:31:52.434223Z","iopub.execute_input":"2026-04-08T09:31:52.434616Z","iopub.status.idle":"2026-04-08T09:31:52.518626Z","shell.execute_reply.started":"2026-04-08T09:31:52.434589Z","shell.execute_reply":"2026-04-08T09:31:52.517939Z"}},"outputs":[],"execution_count":null},{"id":"ad23839e-a824-4667-86a4-fb232f544935","cell_type":"code","source":"# ── CNN training loop (with augmentation support) ────────────────────────────\ndef train_cnn(model, train_loader, val_loader, epochs=40, lr=1e-3,\n              patience=7, augment=False):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-4)\n    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs)\n\n    history = defaultdict(list)\n    best_val_acc, best_state, patience_cnt = 0.0, None, 0\n\n    for epoch in range(1, epochs + 1):\n        model.train()\n        train_loss, correct, total = 0.0, 0, 0\n        for X_batch, y_batch in train_loader:\n            # Reshape flat → image\n            X_batch = X_batch.view(-1, 1, 28, 28).to(DEVICE)\n            y_batch = y_batch.to(DEVICE)\n            if augment:\n                X_batch = aug_transform(X_batch)\n            optimizer.zero_grad()\n            out  = model(X_batch)\n            loss = criterion(out, y_batch)\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item() * len(y_batch)\n            correct    += (out.argmax(1) == y_batch).sum().item()\n            total      += len(y_batch)\n        scheduler.step()\n\n        model.eval()\n        val_loss, val_correct, val_total = 0.0, 0, 0\n        with torch.no_grad():\n            for X_batch, y_batch in val_loader:\n                X_batch = X_batch.view(-1, 1, 28, 28).to(DEVICE)\n                y_batch = y_batch.to(DEVICE)\n                out = model(X_batch)\n                loss = criterion(out, y_batch)\n                val_loss    += loss.item() * len(y_batch)\n                val_correct += (out.argmax(1) == y_batch).sum().item()\n                val_total   += len(y_batch)\n\n        tr_acc  = correct / total\n        val_acc = val_correct / val_total\n        history['train_loss'].append(train_loss / total)\n        history['val_loss'].append(val_loss / val_total)\n        history['train_acc'].append(tr_acc)\n        history['val_acc'].append(val_acc)\n\n        if epoch % 5 == 0:\n            print(f'Epoch {epoch:3d} | Train Acc: {tr_acc:.4f} | '\n                  f'Val Acc: {val_acc:.4f} | Val Loss: {val_loss/val_total:.4f}')\n\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            best_state   = {k: v.clone() for k, v in model.state_dict().items()}\n            patience_cnt = 0\n        else:\n            patience_cnt += 1\n            if patience_cnt >= patience:\n                print(f'Early stopping at epoch {epoch}')\n                break\n\n    model.load_state_dict(best_state)\n    print(f'Best val accuracy: {best_val_acc:.4f}')\n    return history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:31:52.519595Z","iopub.execute_input":"2026-04-08T09:31:52.519887Z","iopub.status.idle":"2026-04-08T09:31:52.529555Z","shell.execute_reply.started":"2026-04-08T09:31:52.519855Z","shell.execute_reply":"2026-04-08T09:31:52.528993Z"}},"outputs":[],"execution_count":null},{"id":"1c5b381d-476f-4bb4-a9b3-443ba43fade7","cell_type":"code","source":"# ── 6.1 Train CNN (baseline) ─────────────────────────────────────────────────\nprint('=== CNN – Baseline (no augmentation) ===')\ncnn = CNN(dropout=0.3).to(DEVICE)\nhistory_cnn = train_cnn(cnn, cnn_train_loader, cnn_val_loader,\n                        epochs=50, lr=1e-3, patience=8)\nplot_history(history_cnn, title='CNN Training Curves', filename='cnn_curves.png')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:31:52.530498Z","iopub.execute_input":"2026-04-08T09:31:52.530774Z","iopub.status.idle":"2026-04-08T09:32:47.141587Z","shell.execute_reply.started":"2026-04-08T09:31:52.530751Z","shell.execute_reply":"2026-04-08T09:32:47.140766Z"}},"outputs":[],"execution_count":null},{"id":"6594ea92-a23c-4ebf-87c9-48717adc62d2","cell_type":"code","source":"# ── 6.2 CNN with data augmentation ───────────────────────────────────────────\nprint('\\n=== CNN – With Data Augmentation ===')\ncnn_aug = CNN(dropout=0.3).to(DEVICE)\nhistory_cnn_aug = train_cnn(cnn_aug, cnn_train_loader, cnn_val_loader,\n                             epochs=50, lr=1e-3, patience=8, augment=True)\n\nfig, ax = plt.subplots(figsize=(8, 4))\nax.plot(history_cnn['val_acc'],     label='No Augmentation')\nax.plot(history_cnn_aug['val_acc'], label='With Augmentation')\nax.set_title('CNN – Effect of Data Augmentation')\nax.set_xlabel('Epoch'); ax.set_ylabel('Validation Accuracy'); ax.legend()\nplt.tight_layout()\nplt.savefig('cnn_augmentation_compare.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:32:47.142749Z","iopub.execute_input":"2026-04-08T09:32:47.143012Z","iopub.status.idle":"2026-04-08T09:33:55.545945Z","shell.execute_reply.started":"2026-04-08T09:32:47.142987Z","shell.execute_reply":"2026-04-08T09:33:55.545348Z"}},"outputs":[],"execution_count":null},{"id":"2badbbdb-d18b-4e1e-b80d-f0d124a10151","cell_type":"code","source":"# ── 6.3 Hyperparameter experiment: batch size ─────────────────────────────────\nprint('\\n=== CNN – Larger batch size (256) ===')\ncnn_bs256_train = make_loader(X_train_img.reshape(len(X_train_img), -1),\n                               y_train, batch_size=256)\ncnn_bs256 = CNN(dropout=0.3).to(DEVICE)\nhistory_cnn_bs = train_cnn(cnn_bs256, cnn_bs256_train, cnn_val_loader,\n                            epochs=50, lr=1e-3, patience=8)\n\nfig, ax = plt.subplots(figsize=(8, 4))\nax.plot(history_cnn['val_acc'],    label='Batch=128')\nax.plot(history_cnn_bs['val_acc'], label='Batch=256')\nax.set_title('CNN – Effect of Batch Size')\nax.set_xlabel('Epoch'); ax.set_ylabel('Validation Accuracy'); ax.legend()\nplt.tight_layout()\nplt.savefig('cnn_batchsize_compare.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:33:55.546772Z","iopub.execute_input":"2026-04-08T09:33:55.546987Z","iopub.status.idle":"2026-04-08T09:35:28.210380Z","shell.execute_reply.started":"2026-04-08T09:33:55.546966Z","shell.execute_reply":"2026-04-08T09:35:28.209690Z"}},"outputs":[],"execution_count":null},{"id":"4a0c4a83-d2b7-4a63-9432-6807835ca1ac","cell_type":"code","source":"# ── 6.4 Evaluate best CNN ─────────────────────────────────────────────────────\ndef evaluate_cnn(model, loader):\n    model.eval()\n    all_preds, all_true = [], []\n    with torch.no_grad():\n        for X_batch, y_batch in loader:\n            X_batch = X_batch.view(-1, 1, 28, 28).to(DEVICE)\n            out = model(X_batch)\n            all_preds.extend(out.argmax(1).cpu().numpy())\n            all_true.extend(y_batch.numpy())\n    return np.array(all_preds), np.array(all_true)\n\n# Pick best CNN (aug vs no-aug)\nbest_cnn = cnn_aug if max(history_cnn_aug['val_acc']) > max(history_cnn['val_acc']) else cnn\n\ncnn_val_preds, _  = evaluate_cnn(best_cnn, cnn_val_loader)\ncnn_val_acc       = accuracy_score(y_val, cnn_val_preds)\ncnn_dig_preds, _  = evaluate_cnn(best_cnn, cnn_dig_loader)\ncnn_dig_acc       = accuracy_score(y_dig, cnn_dig_preds)\n\nprint(f'CNN Validation Accuracy : {cnn_val_acc:.4f}')\nprint(f'CNN Dig-MNIST  Accuracy : {cnn_dig_acc:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:28.211199Z","iopub.execute_input":"2026-04-08T09:35:28.211796Z","iopub.status.idle":"2026-04-08T09:35:28.946927Z","shell.execute_reply.started":"2026-04-08T09:35:28.211770Z","shell.execute_reply":"2026-04-08T09:35:28.946062Z"}},"outputs":[],"execution_count":null},{"id":"3bdf9ce3-0f6d-4aba-8541-64ee7e4cf7a7","cell_type":"markdown","source":"## 7. Model Evaluation & Comparison","metadata":{}},{"id":"f68fd27a-03b0-4b10-8845-a57c69599e77","cell_type":"code","source":"# ── 7.1 Summary comparison table ─────────────────────────────────────────────\nresults = {\n    'Model':         ['KNN', 'SVM', 'Random Forest', 'MLP', 'CNN'],\n    'Val Accuracy':  [knn_val_acc, svm_val_acc, rf_val_acc, mlp_val_acc, cnn_val_acc],\n    'Dig Accuracy':  [knn_dig_acc, svm_dig_acc, rf_dig_acc, mlp_dig_acc, cnn_dig_acc],\n}\nresults_df = pd.DataFrame(results)\nresults_df['Val Accuracy']  = results_df['Val Accuracy'].map('{:.4f}'.format)\nresults_df['Dig Accuracy']  = results_df['Dig Accuracy'].map('{:.4f}'.format)\nprint(results_df.to_string(index=False))\n\n# Bar chart\nfig, ax = plt.subplots(figsize=(10, 4))\nx = np.arange(5)\nwidth = 0.35\nval_accs = [knn_val_acc, svm_val_acc, rf_val_acc, mlp_val_acc, cnn_val_acc]\ndig_accs = [knn_dig_acc, svm_dig_acc, rf_dig_acc, mlp_dig_acc, cnn_dig_acc]\nbars1 = ax.bar(x - width/2, val_accs, width, label='Validation', color='steelblue')\nbars2 = ax.bar(x + width/2, dig_accs, width, label='Dig-MNIST',  color='coral')\nax.set_xticks(x)\nax.set_xticklabels(['KNN', 'SVM', 'Random\\nForest', 'MLP', 'CNN'])\nax.set_ylim(0, 1.05)\nax.set_ylabel('Accuracy')\nax.set_title('Model Accuracy Comparison')\nax.legend()\nfor bar in bars1: ax.text(bar.get_x()+bar.get_width()/2, bar.get_height()+0.005,\n                          f'{bar.get_height():.3f}', ha='center', va='bottom', fontsize=8)\nfor bar in bars2: ax.text(bar.get_x()+bar.get_width()/2, bar.get_height()+0.005,\n                          f'{bar.get_height():.3f}', ha='center', va='bottom', fontsize=8)\nplt.tight_layout()\nplt.savefig('model_comparison.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:28.948016Z","iopub.execute_input":"2026-04-08T09:35:28.948348Z","iopub.status.idle":"2026-04-08T09:35:29.315669Z","shell.execute_reply.started":"2026-04-08T09:35:28.948315Z","shell.execute_reply":"2026-04-08T09:35:29.314860Z"}},"outputs":[],"execution_count":null},{"id":"750e3834-bb82-4585-b120-2f8dca5577a8","cell_type":"code","source":"# ── 7.2 Confusion matrices ────────────────────────────────────────────────────\nmodels_preds = {\n    'KNN':          (knn_val_preds,  y_val),\n    'SVM':          (svm_val_preds,  y_val),\n    'Random Forest':(rf_val_preds,   y_val),\n    'MLP':          (mlp_val_preds,  y_val),\n    'CNN':          (cnn_val_preds,  y_val),\n}\n\nfig, axes = plt.subplots(1, 5, figsize=(28, 5))\nfor ax, (name, (preds, true)) in zip(axes, models_preds.items()):\n    cm = confusion_matrix(true, preds)\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=ax,\n                xticklabels=range(10), yticklabels=range(10), cbar=False)\n    ax.set_title(f'{name}\\nAcc={accuracy_score(true, preds):.4f}', fontsize=10)\n    ax.set_xlabel('Predicted'); ax.set_ylabel('True')\n\nplt.suptitle('Confusion Matrices – Validation Set', fontsize=14, y=1.02)\nplt.tight_layout()\nplt.savefig('confusion_matrices.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:29.316865Z","iopub.execute_input":"2026-04-08T09:35:29.317599Z","iopub.status.idle":"2026-04-08T09:35:32.657532Z","shell.execute_reply.started":"2026-04-08T09:35:29.317571Z","shell.execute_reply":"2026-04-08T09:35:32.656834Z"}},"outputs":[],"execution_count":null},{"id":"81e93204-34f7-4217-8c2b-0f11ab8b3815","cell_type":"code","source":"# ── 7.3 Per-class classification report ──────────────────────────────────────\nfor name, (preds, true) in models_preds.items():\n    print(f'\\n=== {name} ===')  \n    print(classification_report(true, preds, digits=4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:32.658444Z","iopub.execute_input":"2026-04-08T09:35:32.658825Z","iopub.status.idle":"2026-04-08T09:35:32.710203Z","shell.execute_reply.started":"2026-04-08T09:35:32.658789Z","shell.execute_reply":"2026-04-08T09:35:32.709531Z"}},"outputs":[],"execution_count":null},{"id":"90e840a4-b3be-46c5-830e-67d65f5783aa","cell_type":"markdown","source":"## 8. Error Analysis","metadata":{}},{"id":"72d34fce-610e-4532-be43-8e2f25444ae3","cell_type":"code","source":"# ── 8.1 Most confused digit pairs (CNN) ─────────────────────────────────────\ncm_cnn = confusion_matrix(y_val, cnn_val_preds)\nnp.fill_diagonal(cm_cnn, 0)   # remove correct predictions\n\nflat_idx = np.argsort(cm_cnn.ravel())[::-1][:10]\nprint('Top-10 confused pairs (true → predicted | count):')\nfor idx in flat_idx:\n    i, j = divmod(idx, 10)\n    print(f'  {i} → {j}: {cm_cnn[i, j]}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:32.711154Z","iopub.execute_input":"2026-04-08T09:35:32.711543Z","iopub.status.idle":"2026-04-08T09:35:32.718061Z","shell.execute_reply.started":"2026-04-08T09:35:32.711504Z","shell.execute_reply":"2026-04-08T09:35:32.717313Z"}},"outputs":[],"execution_count":null},{"id":"bae4418b-6614-42ae-9851-af0e033d6e36","cell_type":"code","source":"# ── 8.2 Visualize misclassified examples (CNN) ────────────────────────────────\nmisclassified_idx = np.where(cnn_val_preds != y_val)[0]\nnp.random.shuffle(misclassified_idx)\n\nfig, axes = plt.subplots(4, 8, figsize=(16, 8))\nfig.suptitle('CNN – Misclassified Examples', fontsize=14)\nfor ax, idx in zip(axes.ravel(), misclassified_idx[:32]):\n    ax.imshow(X_val[idx].reshape(28, 28), cmap='gray_r')\n    ax.set_title(f'T:{y_val[idx]} P:{cnn_val_preds[idx]}', fontsize=9)\n    ax.axis('off')\nplt.tight_layout()\nplt.savefig('misclassified.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:32.718987Z","iopub.execute_input":"2026-04-08T09:35:32.719256Z","iopub.status.idle":"2026-04-08T09:35:34.616186Z","shell.execute_reply.started":"2026-04-08T09:35:32.719233Z","shell.execute_reply":"2026-04-08T09:35:34.615575Z"}},"outputs":[],"execution_count":null},{"id":"61bf1af6-949a-4151-b78d-d61a2d820f6e","cell_type":"code","source":"# ── 8.3 Visualize confused pairs ─────────────────────────────────────────────\n# Show pairs for the top-3 most confused (true, pred) combinations\ncm_cnn_raw = confusion_matrix(y_val, cnn_val_preds)\nnp.fill_diagonal(cm_cnn_raw, 0)\ntop3 = np.argsort(cm_cnn_raw.ravel())[::-1][:3]\n\nfig, axes = plt.subplots(3, 8, figsize=(16, 6))\nfig.suptitle('Most Confused Digit Pairs', fontsize=13)\nfor row, idx in enumerate(top3):\n    true_d, pred_d = divmod(idx, 10)\n    confused_idx = np.where((y_val == true_d) & (cnn_val_preds == pred_d))[0][:8]\n    for col in range(8):\n        ax = axes[row][col]\n        if col < len(confused_idx):\n            ax.imshow(X_val[confused_idx[col]].reshape(28, 28), cmap='gray_r')\n        ax.axis('off')\n    axes[row][0].set_ylabel(f'True {true_d}\\n→ Pred {pred_d}',\n                            fontsize=10, rotation=0, labelpad=65, va='center')\nplt.tight_layout()\nplt.savefig('confused_pairs.png', dpi=120, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:34.617178Z","iopub.execute_input":"2026-04-08T09:35:34.617526Z","iopub.status.idle":"2026-04-08T09:35:35.411449Z","shell.execute_reply.started":"2026-04-08T09:35:34.617490Z","shell.execute_reply":"2026-04-08T09:35:35.410592Z"}},"outputs":[],"execution_count":null},{"id":"773295a9-9bd6-4e55-a89a-a1fe8f8bd69a","cell_type":"code","source":"# ── 8.4 Model confidence on misclassifications (CNN) ─────────────────────────\nbest_cnn.eval()\nsoftmax_probs = []\nwith torch.no_grad():\n    for X_batch, _ in cnn_val_loader:\n        X_batch = X_batch.view(-1, 1, 28, 28).to(DEVICE)\n        out = best_cnn(X_batch)\n        probs = F.softmax(out, dim=1)\n        softmax_probs.append(probs.cpu().numpy())\nsoftmax_probs = np.concatenate(softmax_probs)\n\ncorrect_conf   = softmax_probs[cnn_val_preds == y_val].max(axis=1)\nincorrect_conf = softmax_probs[cnn_val_preds != y_val].max(axis=1)\n\nfig, ax = plt.subplots(figsize=(8, 4))\nax.hist(correct_conf,   bins=50, alpha=0.6, label='Correct',   color='steelblue')\nax.hist(incorrect_conf, bins=50, alpha=0.6, label='Incorrect',  color='coral')\nax.set_title('CNN – Confidence Distribution: Correct vs Misclassified')\nax.set_xlabel('Max Softmax Probability'); ax.set_ylabel('Count')\nax.legend()\nplt.tight_layout()\nplt.savefig('confidence_dist.png', dpi=120, bbox_inches='tight')\nplt.show()\nprint(f'Mean confidence – Correct: {correct_conf.mean():.4f} | Incorrect: {incorrect_conf.mean():.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:35.412694Z","iopub.execute_input":"2026-04-08T09:35:35.412965Z","iopub.status.idle":"2026-04-08T09:35:36.309268Z","shell.execute_reply.started":"2026-04-08T09:35:35.412940Z","shell.execute_reply":"2026-04-08T09:35:36.308511Z"}},"outputs":[],"execution_count":null},{"id":"994c75ed-a1bc-44d2-b8c6-d61fc0314341","cell_type":"markdown","source":"## 9. Kaggle Submission","metadata":{}},{"id":"b7e13208-7a3a-4fea-97c0-3bf0c60263c1","cell_type":"code","source":"# ── Generate submission.csv using best CNN ────────────────────────────────────\ntest_loader = make_loader(\n    X_test_img.reshape(len(X_test_img), -1),\n    np.zeros(len(X_test_img), dtype=int),   # dummy labels\n    batch_size=256, shuffle=False\n)\n\nbest_cnn.eval()\ntest_preds = []\nwith torch.no_grad():\n    for X_batch, _ in test_loader:\n        X_batch = X_batch.view(-1, 1, 28, 28).to(DEVICE)\n        out = best_cnn(X_batch)\n        test_preds.extend(out.argmax(1).cpu().numpy())\n\nsubmission = pd.DataFrame({'id': test_df['id'], 'label': test_preds})\nsubmission.to_csv('submission.csv', index=False)\nprint('Saved submission.csv')\nsubmission.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-08T09:35:36.310562Z","iopub.execute_input":"2026-04-08T09:35:36.310864Z","iopub.status.idle":"2026-04-08T09:35:36.505492Z","shell.execute_reply.started":"2026-04-08T09:35:36.310839Z","shell.execute_reply":"2026-04-08T09:35:36.504629Z"}},"outputs":[],"execution_count":null}]}