{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":16017,"databundleVersionId":708189}],"dockerImageVersionId":31329,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Kannada-MNIST Digit Classification: Classical ML to Deep Learning Ensemble\n**Objective:** The goal of this project is to accurately classify handwritten digits of the Kannada language. We will explore the data, establish baselines, and progressively build complex Convolutional Neural Networks (ResNet, EfficientNet, DenseNet). Finally, we will combine the strongest models into an ensemble and evaluate their robustness on an out-of-domain dataset (Dig-MNIST).","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score, recall_score, precision_score, roc_auc_score, roc_curve, auc, classification_report, confusion_matrix, ConfusionMatrixDisplay\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom torchvision import transforms, models\nfrom PIL import Image\nimport torch.nn.functional as F\n\n!pip install optuna\nimport optuna\nfrom optuna import Trial, visualization\nfrom optuna.samplers import TPESampler\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nRANDOM_STATE = 42\nEPOCHS = 20\nKFOLD = KFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)\nOPTUNA_TRIALS = 2\nmodel_results = pd.DataFrame(columns=[\"Model name\", \"Accuracy\", \"Balanced Accuracy\", \"F1 Score\", \"Recall\", \"Precision\", \"ROC AUC Score\"])\n\nrandom.seed(RANDOM_STATE)\nos.environ['PYTHONHASHSEED'] = str(RANDOM_STATE)\nnp.random.seed(RANDOM_STATE)\ntorch.manual_seed(RANDOM_STATE)\ntorch.cuda.manual_seed(RANDOM_STATE)\ntorch.cuda.manual_seed_all(RANDOM_STATE)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"{device}\")","metadata":{"id":"FjNFR8MWwVj6","outputId":"beaccef4-6fc3-4824-e90e-7b573e530bc9"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/competitions/Kannada-MNIST\"\n\nif not os.path.exists(DATA_PATH):\n  DATA_PATH = '.'\n\ndf_train = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\ndf_test = pd.read_csv(os.path.join(DATA_PATH, \"test.csv\"))\ndf_dig = pd.read_csv(os.path.join(DATA_PATH, \"Dig-MNIST.csv\"))\n\nX = df_train.drop(\"label\", axis=1).values.astype(np.float32)\ny = df_train[\"label\"].values\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=RANDOM_STATE, stratify=y)\n\nX_test_kaggle = df_test.drop(\"id\", axis=1).values.astype(np.float32)\n\nX_dig = df_dig.drop(\"label\", axis=1).values.astype(np.float32)\ny_dig = df_dig[\"label\"].values\n\nprint(f\"Training set: {X_train.shape[0]} samples\")\nprint(f\"Validation set: {X_val.shape[0]} samples\")\nprint(f\"Kaggle test set: {X_test_kaggle.shape[0]} samples\")\nprint(f\"Dig-MNIST set: {X_dig.shape[0]} samples\")","metadata":{"id":"BsxpoKYplsV3","outputId":"52c1f8de-50a0-4f87-a5bb-e4dececcccd7","jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape, y_train.shape, X_val.shape, y_val.shape, X_test_kaggle.shape, X_dig.shape, y_dig.shape","metadata":{"id":"oVTDT4aoaVLW","outputId":"31dc28ab-c482-43e3-fa40-054906ce81d0"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Class distribution in Kannada-MNIST set\n\nprint(\"Kannada-MNIST:\\n\")\n\nplt.figure(figsize=(8, 4))\nsns.countplot(x=y_train)\nplt.title(\"Class distribution in training set\")\nplt.xlabel(\"Digit class\")\nplt.ylabel(\"Count\")\nplt.show()\n\nprint()\n\nX_train_img = X_train.reshape(-1, 28, 28)\nfig, axes = plt.subplots(2, 5, figsize=(12, 5))\nfig.suptitle(\"Sample images from Kannada-MNIST\", fontsize=16)\nfor i in range(10):\n  ax = axes[i//5, i%5]\n  img_idx = np.where(y_train == i)[0][0]\n  ax.imshow(X_train_img[img_idx], cmap=\"gray\")\n  ax.set_title(f\"Class: {i}\")\n  ax.axis(\"off\")\nplt.tight_layout()\nplt.show()\n\nprint(\"Dig-MNIST:\\n\")\n\n# Class distribution in Dig-MNIST set\n\nplt.figure(figsize=(8, 4))\nsns.countplot(x=y_dig)\nplt.title(\"Class distribution in Dig-MNIST set\")\nplt.xlabel(\"Digit class\")\nplt.ylabel(\"Count\")\nplt.show()\n\nprint()\n\nX_dig_img = X_dig.reshape(-1, 28, 28)\nfig, axes = plt.subplots(2, 5, figsize=(12, 5))\nfig.suptitle(\"Sample images from Dig-MNIST\", fontsize=16)\nfor i in range(10):\n  ax = axes[i//5, i%5]\n  img_idx = np.where(y_train == i)[0][0]\n  ax.imshow(X_dig_img[img_idx], cmap=\"gray\")\n  ax.set_title(f\"Class: {i}\")\n  ax.axis(\"off\")\nplt.tight_layout()\nplt.show()","metadata":{"id":"ZZ7U59slrUAX","outputId":"b68f757f-d4b8-4546-8d01-6854d40eecca"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def eval_func(model, model_name, X_test, y_test, model_results, X_train=None, y_train=None, fit_model=True):\n  if fit_model:\n    model.fit(X_train, y_train)\n\n  y_pred = model.predict(X_test)\n  y_pred_proba = model.predict_proba(X_test)\n\n  print(f\"Accuracy: {accuracy_score(y_test, y_pred):.4f}\")\n  print(f\"Balanced Accuracy: {balanced_accuracy_score(y_test, y_pred):.4f}\")\n  print(f\"F1 score: {f1_score(y_test, y_pred, average=\"macro\"):.4f}\")\n  print(f\"Recall: {recall_score(y_test, y_pred, average=\"macro\"):.4f}\")\n  print(f\"Precision: {precision_score(y_test, y_pred, average=\"macro\"):.4f}\")\n  print(f\"ROC AUC Score: {roc_auc_score(y_test, y_pred_proba, multi_class=\"ovr\"):.4f}\\n\")\n\n  fig, ax = plt.subplots(figsize=(8, 6))\n  ConfusionMatrixDisplay.from_predictions(\n    y_test,\n    y_pred,\n    cmap=\"Blues\",\n    ax=ax\n  )\n  plt.title(f\"Confusion matrix: {model_name}\")\n  plt.show()\n\n  curr_results = {\n    \"Model name\": model_name,\n    \"Accuracy\": accuracy_score(y_test, y_pred),\n    \"Balanced Accuracy\": balanced_accuracy_score(y_test, y_pred),\n    \"F1 Score\": f1_score(y_test, y_pred, average=\"macro\"),\n    \"Recall\": recall_score(y_test, y_pred, average=\"macro\"),\n    \"Precision\": precision_score(y_test, y_pred, average=\"macro\"),\n    \"ROC AUC Score\": roc_auc_score(y_test, y_pred_proba, multi_class=\"ovr\")\n  }\n\n  model_results = pd.concat([model_results, pd.DataFrame([curr_results])], ignore_index=True)\n  return model_results","metadata":{"id":"QBxOfLtgiZMF"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Baseline Random Forest","metadata":{"id":"Ken-vpEheHNa"}},{"cell_type":"code","source":"# Baseline Random Forest\n\nrf_clf = RandomForestClassifier(n_jobs=-1, random_state=RANDOM_STATE)\n\nmodel_name = \"Baseline Random Forest Kannada-MNIST\"\nprint(\"Model scores on Kannada-MNIST:\")\nmodel_results = eval_func(rf_clf, model_name, X_val, y_val, model_results, X_train, y_train, fit_model=True)\n\nmodel_name = \"Baseline Random Forest Dig-MNIST\"\nprint(\"\\nModel scores on Dig-MNIST:\")\nmodel_results = eval_func(rf_clf, model_name, X_dig, y_dig, model_results, fit_model=False)\n\nmodel_results","metadata":{"id":"PJZeMrNseK9y","outputId":"de5797f4-13e7-49c1-88eb-9db364864752"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Baseline K-Nearest Neighbors","metadata":{"id":"FAnEmWOnmqnb"}},{"cell_type":"code","source":"# Baseline K-Nearest Neighbors\n\nknn = KNeighborsClassifier(n_jobs=-1)\n\nmodel_name = \"Baseline KNN Kannada-MNIST\"\nprint(\"Model scores on Kannada-MNIST:\")\nmodel_results = eval_func(knn, model_name, X_val, y_val, model_results, X_train, y_train, fit_model=True)\n\nmodel_name = \"Baseline KNN Dig-MNIST\"\nprint(\"\\nModel scores on Dig-MNIST:\")\nmodel_results = eval_func(knn, model_name, X_dig, y_dig, model_results, fit_model=False)\n\nmodel_results","metadata":{"id":"RSSkYsyqZAXv","outputId":"087eb8bf-abc1-47a7-d91e-494fddcde3b2"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Classical models hyperparameter tuning with Optuna","metadata":{"id":"KCrrPkXcphI_"}},{"cell_type":"markdown","source":"### Random Forest","metadata":{"id":"Ge0PLOUtqNOo"}},{"cell_type":"code","source":"def objective_rf_clf(trial):\n  params = {\n    \"n_estimators\" : trial.suggest_int(\"n_estimators\", 50, 150),\n    \"max_depth\" : trial.suggest_int(\"max_depth\", 5, 20),\n    \"min_samples_split\" : trial.suggest_int(\"min_samples_split\", 2, 10),\n    \"min_samples_leaf\" : trial.suggest_int(\"min_samples_leaf\", 1, 5),\n    \"max_features\" : trial.suggest_categorical(\"max_features\", [\"sqrt\", \"log2\"]),\n    \"random_state\" : RANDOM_STATE,\n    \"n_jobs\": -1\n  }\n\n  scores = []\n  X_arr = np.array(X_train)\n  y_arr = np.array(y_train)\n\n  for train_index, val_index in KFOLD.split(X_arr, y_arr):\n    X_train_fold = X_arr[train_index]\n    y_train_fold = y_arr[train_index]\n\n    X_val_fold = X_arr[val_index]\n    y_val_fold = y_arr[val_index]\n\n    model = RandomForestClassifier(**params)\n    model.fit(X_train_fold, y_train_fold)\n\n    y_pred = model.predict(X_val_fold)\n    scores.append(accuracy_score(y_val_fold, y_pred))\n\n  return np.mean(scores)\n\nstudy_rf = optuna.create_study(direction=\"maximize\", sampler=TPESampler(seed=RANDOM_STATE))\nstudy_rf.optimize(objective_rf_clf, n_trials=OPTUNA_TRIALS)\n\nbest_rf_params = study_rf.best_params\nrf_clf_optuna = RandomForestClassifier(**best_rf_params, random_state=RANDOM_STATE, n_jobs=-1)\n\nmodel_name = \"Random Forest (Optuna) Kannada-MNIST\"\nprint(\"\\nModel scores on Kannada-MNIST:\")\nmodel_results = eval_func(rf_clf_optuna, model_name, X_val, y_val, model_results, X_train, y_train, fit_model=True)\n\nmodel_name = \"Random Forest Dig-MNIST\"\nprint(\"\\nModel scores on Dig-MNIST:\")\nmodel_results = eval_func(rf_clf_optuna, model_name, X_dig, y_dig, model_results, fit_model=False)\n\nmodel_results","metadata":{"id":"eLN7KVuLp0iD","outputId":"d68dc8f4-d154-4c93-b8a9-f02962f7146c"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### K-Nearest Neighbors","metadata":{"id":"ovBHIs1ER7Zk"}},{"cell_type":"code","source":"def objective_knn_clf(trial):\n  params = {\n    \"n_neighbors\": trial.suggest_int(\"n_neighbors\", 3, 15),\n    \"weights\": trial.suggest_categorical(\"weights\", [\"uniform\", \"distance\"]),\n    \"p\": trial.suggest_int(\"p\", 1, 2),\n    \"n_jobs\": -1\n  }\n\n  scores = []\n  X_arr = np.array(X_train)\n  y_arr = np.array(y_train)\n\n  for train_index, val_index in KFOLD.split(X_arr, y_arr):\n    X_train_fold = X_arr[train_index]\n    y_train_fold = y_arr[train_index]\n\n    X_val_fold = X_arr[val_index]\n    y_val_fold = y_arr[val_index]\n\n    model = KNeighborsClassifier(**params)\n    model.fit(X_train_fold, y_train_fold)\n\n    y_pred = model.predict(X_val_fold)\n    scores.append(accuracy_score(y_val_fold, y_pred))\n\n  return np.mean(scores)\n\nstudy_knn = optuna.create_study(direction=\"maximize\", sampler=TPESampler(seed=RANDOM_STATE))\nstudy_knn.optimize(objective_knn_clf, n_trials=OPTUNA_TRIALS)\n\nbest_knn_params = study_knn.best_params\nknn_clf_optuna = KNeighborsClassifier(**best_knn_params, n_jobs=-1)\n\nmodel_name = \"KNN (Optuna) Kannada-MNIST\"\nprint(\"Model scores on Kannada-MNIST:\")\nmodel_results = eval_func(knn_clf_optuna, model_name, X_val, y_val, model_results, X_train, y_train, fit_model=True)\n\nmodel_name = \"KNN (Optuna) Dig-MNIST\"\nprint(\"\\nModel scores on Dig-MNIST:\")\nmodel_results = eval_func(knn_clf_optuna, model_name, X_dig, y_dig, model_results, fit_model=False)\n\nmodel_results","metadata":{"id":"YBWYU6EQShrx","outputId":"b0f83475-26dd-417e-9a6a-7df149556d03"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Dataset Definition & Aggressive Augmentation\nStandard PyTorch models cannot read raw CSV arrays directly. Here, we define a custom `Datasets` class that reshapes our 1D arrays into 28x28 2D images. \n\nTo prevent our neural networks from overfitting and to improve their generalization, we apply **Data Augmentation** (`RandomRotation`, `RandomAffine`). Furthermore, we implement a `WeightedRandomSampler` to guarantee that every training batch is perfectly balanced across all 10 digit classes.","metadata":{}},{"cell_type":"code","source":"class Datasets(Dataset):\n  def __init__(self, X, y, transform=None):\n    self.X = X.reshape(-1, 28, 28).astype(np.uint8)\n    self.y = y\n    self.transform = transform\n\n  def __len__(self):\n    return len(self.X)\n\n  def __getitem__(self, idx):\n    image = Image.fromarray(self.X[idx])\n    label = self.y[idx]\n    if self.transform: image = self.transform(image)\n    return image, torch.tensor(label, dtype=torch.long)\n\ntrain_transforms = transforms.Compose([\n  transforms.RandomRotation(10),\n  transforms.RandomAffine(degrees=0, translate=(0.1, 0.1), scale=(0.9, 1.1)),\n  transforms.ToTensor(),\n  transforms.Normalize((0.5,), (0.5,))\n])\n\neval_transforms = transforms.Compose([\n  transforms.ToTensor(),\n  transforms.Normalize((0.5,), (0.5,))\n])\n\ng = torch.Generator()\ng.manual_seed(RANDOM_STATE)\n\nclass_counts = np.bincount(y_train)\nsample_weights = (1.0 / class_counts)[y_train]\nsampler = WeightedRandomSampler(weights=sample_weights, num_samples=len(sample_weights), replacement=True, generator=g)\n\ntrain_loader = DataLoader(Datasets(X_train, y_train, transform=train_transforms), batch_size=128, sampler=sampler, num_workers=0, generator=g)\nval_loader = DataLoader(Datasets(X_val, y_val, transform=eval_transforms), batch_size=128, shuffle=False, num_workers=0)\ndig_loader = DataLoader(Datasets(X_dig, y_dig, transform=eval_transforms), batch_size=128, shuffle=False, num_workers=0)","metadata":{"id":"QFDQNhBiIUqz"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MLP(nn.Module):\n  def __init__(self):\n    super(MLP, self).__init__()\n    self.flatten = nn.Flatten()\n    self.fc1 = nn.Linear(28 * 28, 512)\n    self.dropout1 = nn.Dropout(0.3)\n    self.fc2 = nn.Linear(512, 256)\n    self.dropout2 = nn.Dropout(0.3)\n    self.fc3 = nn.Linear(256, 10)\n    self.relu = nn.ReLU()\n\n  def forward(self, x):\n    x = self.flatten(x)\n    x = self.dropout1(self.relu(self.fc1(x)))\n    x = self.dropout2(self.relu(self.fc2(x)))\n    return self.fc3(x)\n\nclass CNN(nn.Module):\n  def __init__(self):\n    super(CNN, self).__init__()\n    self.conv1 = nn.Conv2d(1, 32, kernel_size=3, padding=1)\n    self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n    self.pool = nn.MaxPool2d(2, 2)\n    self.fc1 = nn.Linear(64 * 7 * 7, 128)\n    self.fc2 = nn.Linear(128, 10)\n    self.relu = nn.ReLU()\n    self.dropout = nn.Dropout(0.25)\n\n  def forward(self, x):\n    x = self.pool(self.relu(self.conv1(x)))\n    x = self.pool(self.relu(self.conv2(x)))\n    x = torch.flatten(x, 1)\n    x = self.dropout(self.relu(self.fc1(x)))\n    return self.fc2(x)\n\ndef resnet18():\n  model = models.resnet18(weights=None)\n  model.conv1 = nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1, bias=False)\n  model.maxpool = nn.Identity()\n  model.fc = nn.Linear(model.fc.in_features, 10)\n\n  return model\n\ndef efficientnet():\n  model = models.efficientnet_b0(weights=None)\n  model.features[0][0] = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1, bias=False)\n  model.classifier[1] = nn.Linear(model.classifier[1].in_features, 10)\n\n  return model\n\ndef densenet():\n  model = models.densenet121(weights=None)\n  model.features.conv0 = nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1, bias=False)\n  model.features.pool0 = nn.Identity()\n  model.classifier = nn.Linear(model.classifier.in_features, 10)\n\n  return model","metadata":{"id":"YeWFIY-UrZTR"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Model Training Engine\nBelow is the core training function used for all our Neural Networks. It utilizes the `AdamW` optimizer and PyTorch's `OneCycleLR` learning rate scheduler. The scheduler dynamically adjusts the learning rate during training, allowing for faster convergence and preventing the model from getting stuck in local minima.\n\n## 4. Visualizing Model Performance\nAfter training our models (MLP, Simple CNN, ResNet18, EfficientNet-B0, DenseNet121), it is crucial to analyze their learning curves (Loss and Accuracy) and their Confusion Matrices to see exactly which digits they struggle to differentiate.\n\n*Disclaimer: The Python code utilized below to generate these visual plots, confusion matrices and comments was written with the assistance of AI.*","metadata":{}},{"cell_type":"code","source":"def train_model(model, train_loader, val_loader, epochs=EPOCHS, max_lr=0.01):\n  model = model.to(device)\n  criterion = nn.CrossEntropyLoss()\n  optimizer = optim.AdamW(model.parameters(), lr=max_lr/10, weight_decay=1e-4)\n  scheduler = optim.lr_scheduler.OneCycleLR(optimizer, max_lr=max_lr, steps_per_epoch=len(train_loader), epochs=epochs)\n\n  history = {\"train_loss\": [], \"val_loss\": [], \"val_acc\": []}\n\n  for epoch in range(epochs):\n    model.train()\n    running_loss = 0.0\n\n    for inputs, labels in train_loader:\n      inputs, labels = inputs.to(device), labels.to(device)\n      optimizer.zero_grad()\n      outputs = model(inputs)\n      loss = criterion(outputs, labels)\n      loss.backward()\n      optimizer.step()\n      scheduler.step()\n      running_loss += loss.item()\n\n    train_loss = running_loss / len(train_loader)\n\n    model.eval()\n    val_loss, correct, total = 0.0, 0, 0\n    with torch.no_grad():\n      for inputs, labels in val_loader:\n        inputs, labels = inputs.to(device), labels.to(device)\n        outputs = model(inputs)\n        val_loss += criterion(outputs, labels).item()\n        _, predicted = outputs.max(1)\n        total += labels.size(0)\n        correct += predicted.eq(labels).sum().item()\n\n    val_loss /= len(val_loader)\n    val_acc = correct / total\n\n    history['train_loss'].append(train_loss)\n    history['val_loss'].append(val_loss)\n    history['val_acc'].append(val_acc)\n    print(f\"Epoch {epoch+1:02d}/{epochs} -- Train loss: {train_loss:.4f} -- Val loss: {val_loss:.4f} -- Val acc: {val_acc*100:.2f}%\")\n\n  return history, model\n\ndef plot_curves(history, title):\n  fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\n  ax1.plot(history[\"train_loss\"], label=\"Train\")\n  ax1.plot(history[\"val_loss\"], label=\"Val\")\n  ax1.set_title(f\"{title} - loss\")\n  ax1.legend()\n\n  ax2.plot(history[\"val_acc\"], label=\"Val acc\", color=\"green\")\n  ax2.set_title(f\"{title} - accuracy\")\n  ax2.legend()\n  plt.show()\n\ndef plot_confusion_matrix(model, data_loader, title):\n    model.eval()\n    all_preds = []\n    all_labels = []\n\n    with torch.no_grad():\n        for inputs, labels in data_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            _, predicted = outputs.max(1)\n\n            all_preds.extend(predicted.cpu().numpy())\n            all_labels.extend(labels.cpu().numpy())\n\n    conf_matrix = confusion_matrix(all_labels, all_preds)\n\n    plt.figure(figsize=(8, 6))\n    sns.heatmap(conf_matrix, annot=True, fmt=\"d\", cmap=\"Blues\")\n    plt.title(f\"{title} - confusion matrix\")\n    plt.xlabel(\"Predicted label\")\n    plt.ylabel(\"True label\")\n    plt.show()","metadata":{"id":"psDcqwf-wSIY"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training on Kannada-MNIST","metadata":{"id":"IoTfLWEUgPP-"}},{"cell_type":"code","source":"print(\"1. MLP training (Kannada-MNIST)\\n\")\nmlp_history, mlp_model = train_model(MLP(), train_loader, val_loader, epochs=EPOCHS, max_lr=0.005)\nprint(\"\\n2. Training CNN (Kannada-MNIST)\\n\")\ncnn_history, cnn_model = train_model(CNN(), train_loader, val_loader, epochs=EPOCHS, max_lr=0.01)\nprint(\"\\n3. ResNet18 training (Kannada-MNIST)\\n\")\nresnet_history, resnet_model = train_model(resnet18(), train_loader, val_loader, epochs=EPOCHS, max_lr=0.01)\nprint(\"\\n4. EfficientNet-B0 training (Kannada-MNIST)\\n\")\nefficientnet_history, efficientnet_model = train_model(efficientnet(), train_loader, val_loader, epochs=EPOCHS, max_lr=0.01)\nprint(\"\\n5. DenseNet121 training (Kannada-MNIST)\\n\")\ndensenet_history, densenet_model = train_model(densenet(), train_loader, val_loader, epochs=EPOCHS, max_lr=0.01)\n\nprint(\"\\nPlotting training curves\\n\")\nplot_curves(mlp_history, \"MLP (Kannada-MNIST)\")\nplot_confusion_matrix(mlp_model, val_loader, \"MLP (Kannada-MNIST)\")\nprint()\nplot_curves(cnn_history, \"CNN (Kannada-MNIST)\")\nplot_confusion_matrix(cnn_model, val_loader, \"CNN (Kannada-MNIST)\")\nprint()\nplot_curves(resnet_history, \"ResNet18 (Kannada-MNIST)\")\nplot_confusion_matrix(resnet_model, val_loader, \"ResNet18 (Kannada-MNIST)\")\nprint()\nplot_curves(efficientnet_history, \"EfficientNet-B0 (Kannada-MNIST)\")\nplot_confusion_matrix(efficientnet_model, val_loader, \"EfficientNet-B0 (Kannada-MNIST)\")\nprint()\nplot_curves(densenet_history, \"DenseNet121 (Kannada-MNIST)\")\nplot_confusion_matrix(densenet_model, val_loader, \"DenseNet121 (Kannada-MNIST)\")","metadata":{"id":"LXVwoW-QIaUq","outputId":"15cfb28b-7fa9-48c1-8306-1d7e493d26be"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluation on Dig-MNIST\n\n## 5. Cross-Domain Evaluation (Dig-MNIST)\nThe `Dig-MNIST` dataset serves as an excellent out-of-distribution test. It contains Kannada digits written by different native speakers in a slightly different style. Testing our models here tells us how well they truly generalize to the real world, rather than just memorizing the training set.","metadata":{"id":"C-G-idSagLaH"}},{"cell_type":"code","source":"def evaluate_on_dig(model, loader, model_name):\n  model.eval()\n  correct = 0\n  total = 0\n\n  with torch.no_grad():\n    for inputs, labels in loader:\n      inputs, labels = inputs.to(device), labels.to(device)\n      outputs = model(inputs)\n      _, predicted = outputs.max(1)\n      total += labels.size(0)\n      correct += predicted.eq(labels).sum().item()\n\n  accuracy = correct / total\n  print(f\"{model_name} Accuracy (Dig-MNIST): {accuracy*100:.2f}%\")\n\n  return accuracy\n\nprint()\nevaluate_on_dig(mlp_model, dig_loader, \"MLP (Dig-MNIST)\")\nplot_confusion_matrix(mlp_model, dig_loader, \"MLP (Dig-MNIST)\")\nprint()\nevaluate_on_dig(cnn_model, dig_loader, \"CNN (Dig-MNIST)\")\nplot_confusion_matrix(cnn_model, dig_loader, \"CNN (Dig-MNIST)\")\nprint()\nevaluate_on_dig(resnet_model, dig_loader, \"ResNet18 (Dig-MNIST)\")\nplot_confusion_matrix(resnet_model, dig_loader, \"ResNet18 (Dig-MNIST)\")\nprint()\nevaluate_on_dig(efficientnet_model, dig_loader, \"EfficientNet-B0 (Dig-MNIST)\")\nplot_confusion_matrix(efficientnet_model, dig_loader, \"EfficientNet-B0 (Dig-MNIST)\")\nprint()\nevaluate_on_dig(densenet_model, dig_loader, \"DenseNet121 (Dig-MNIST)\")\nplot_confusion_matrix(densenet_model, dig_loader, \"DenseNet121 (Dig-MNIST)\")\nprint()","metadata":{"id":"hKZOGH46gKn9"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ensemble model on Kannada-MNIST\n\n## 6. The 3-Model Ensemble (Soft-Voting)\nTo achieve maximum accuracy, we combine our three most powerful Deep Learning models: **ResNet18, EfficientNet-B0, and DenseNet121**. \n\nInstead of taking a simple majority vote (hard voting), we use **Soft-Voting**. We extract the raw Softmax probabilities from each model for a given image, average them together, and select the digit with the highest combined confidence. This method successfully smooths out the individual biases of each network.","metadata":{"id":"J8iWmTcJgwFC"}},{"cell_type":"code","source":"print(\"Ensemble model ResNet18, EfficientNet, DenseNet (Kannada-MNIST)\\n\")\n\nensemble_models = [resnet_model, efficientnet_model, densenet_model]\nfor model in ensemble_models: model.eval()\n\nensemble_preds, test_labels_arr = [], []\n\nwith torch.no_grad():\n  for inputs, labels in val_loader:\n    inputs = inputs.to(device)\n    summed_probs = torch.zeros((inputs.shape[0], 10)).to(device)\n\n    for model in ensemble_models:\n      out = model(inputs)\n      summed_probs += F.softmax(out, dim=1)\n\n    avg_probs = summed_probs / len(ensemble_models)\n    _, final_preds = avg_probs.max(1)\n\n    ensemble_preds.extend(final_preds.cpu().numpy())\n    test_labels_arr.extend(labels.numpy())\n\nensemble_preds = np.array(ensemble_preds)\ntest_labels_arr = np.array(test_labels_arr)\n\nensemble_acc = accuracy_score(test_labels_arr, ensemble_preds)\nprint(f\"Ensemble accuracy: {ensemble_acc * 100:.4f}%\")\n\nplt.figure(figsize=(10, 8))\nconf_matrix = confusion_matrix(test_labels_arr, ensemble_preds)\nsns.heatmap(conf_matrix, annot=True, fmt=\"d\", cmap=\"Greens\")\nplt.title(\"Ensemble confusion matrix\")\nplt.xlabel(\"Predicted label\")\nplt.ylabel(\"True label\")\nplt.show()\n\nerrors_idx = np.where(ensemble_preds != test_labels_arr)[0]\nprint(f\"Misclassified images in test set: {len(errors_idx)}\")\n\nif len(errors_idx) > 0:\n  fig, axes = plt.subplots(2, 5, figsize=(15, 6))\n  fig.suptitle(\"Misclassified examples (predicted vs. actual)\", fontsize=16)\n\n  for i in range(min(10, len(errors_idx))):\n    idx = errors_idx[i]\n    ax = axes[i//5, i%5]\n    img = X_val[idx].reshape(28, 28)\n    ax.imshow(img, cmap=\"gray\")\n    ax.set_title(f\"Predicted: {ensemble_preds[idx]} vs. actual: {test_labels_arr[idx]}\")\n    ax.axis(\"off\")\n\n  plt.tight_layout()\n  plt.show()","metadata":{"id":"k8YvtAmDIdBV"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ensemble model evaluation on Dig-MNIST","metadata":{"id":"H932e3Lcg07n"}},{"cell_type":"code","source":"print(\"Ensemble model ResNet18, EfficientNet-B0, DenseNet121 (Dig-MNIST)\\n\")\n\nensemble_models = [resnet_model, efficientnet_model, densenet_model]\nfor model in ensemble_models: model.eval()\n\nensemble_preds_dig, dig_labels_arr = [], []\n\nwith torch.no_grad():\n  for inputs, labels in dig_loader:\n    inputs = inputs.to(device)\n    summed_probs = torch.zeros((inputs.shape[0], 10)).to(device)\n\n    for model in ensemble_models:\n      out = model(inputs)\n      summed_probs += F.softmax(out, dim=1)\n\n    avg_probs = summed_probs / len(ensemble_models)\n    _, final_preds = avg_probs.max(1)\n\n    ensemble_preds_dig.extend(final_preds.cpu().numpy())\n    dig_labels_arr.extend(labels.numpy())\n\nensemble_preds_dig = np.array(ensemble_preds_dig)\ndig_labels_arr = np.array(dig_labels_arr)\n\nensemble_acc_dig = accuracy_score(dig_labels_arr, ensemble_preds_dig)\nprint(f\"Ensemble accuracy (Dig-MNIST): {ensemble_acc_dig * 100:.4f}%\")\n\nplt.figure(figsize=(10, 8))\nconf_matrix_dig = confusion_matrix(dig_labels_arr, ensemble_preds_dig)\nsns.heatmap(conf_matrix_dig, annot=True, fmt=\"d\", cmap=\"Blues\")\nplt.title(\"Ensemble confusion matrix (Dig-MNIST)\")\nplt.xlabel(\"Predicted label\")\nplt.ylabel(\"True label\")\nplt.show()\n\nerrors_idx_dig = np.where(ensemble_preds_dig != dig_labels_arr)[0]\nprint(f\"Misclassified images in Dig-MNIST set: {len(errors_idx_dig)}\")\n\nif len(errors_idx_dig) > 0:\n  fig, axes = plt.subplots(2, 5, figsize=(15, 6))\n  fig.suptitle(\"Misclassified examples in Dig-MNIST (predicted vs. actual)\", fontsize=16)\n\n  for i in range(min(10, len(errors_idx_dig))):\n    idx = errors_idx_dig[i]\n    ax = axes[i//5, i%5]\n    img = X_dig[idx].reshape(28, 28)\n    ax.imshow(img, cmap=\"gray\")\n    ax.set_title(f\"Pred: {ensemble_preds_dig[idx]} vs. Act: {dig_labels_arr[idx]}\")\n    ax.axis(\"off\")\n\n  plt.tight_layout()\n  plt.show()","metadata":{"id":"9o0N3MdKg0vk"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Generating Kaggle Submission\nFinally, we apply our highly accurate 3-Model Ensemble to the Kaggle blind test set (`test.csv`). Since this dataset lacks labels, we pass an array of dummy labels to our DataLoader to prevent errors. The final predictions are extracted and formatted perfectly into `submission.csv` to be scored by the Kaggle leaderboard.","metadata":{"id":"X1e7_6NZjEI0"}},{"cell_type":"code","source":"dummy_labels = np.zeros(X_test_kaggle.shape[0])\nkaggle_test_loader = DataLoader(Datasets(X_test_kaggle, dummy_labels, transform=eval_transforms), batch_size=128, shuffle=False, num_workers=0)\nensemble_models = [resnet_model, efficientnet_model, densenet_model]\n\nfor model in ensemble_models:\n  model.eval()\n\nkaggle_preds = []\n\nwith torch.no_grad():\n  for inputs, _ in kaggle_test_loader:\n    inputs = inputs.to(device)\n    summed_probs = torch.zeros((inputs.shape[0], 10)).to(device)\n\n    for model in ensemble_models:\n      out = model(inputs)\n      summed_probs += F.softmax(out, dim=1)\n\n    avg_probs = summed_probs / len(ensemble_models)\n    _, final_preds = avg_probs.max(1)\n\n    kaggle_preds.extend(final_preds.cpu().numpy())\n\nsubmission = pd.DataFrame({\n  \"id\" : df_test[\"id\"],\n  \"label\" : kaggle_preds\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission","metadata":{"id":"Lb-u_04nIxl1"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"c6GmfFgojgmp"},"outputs":[],"execution_count":null}]}