{"cells": [{"cell_type": "markdown", "id": "383c65f9", "metadata": {}, "source": "# Digit Recognizer: Official 5-Seed CNN TTA\n\nThis is a clean modeling-only route for the Kaggle Digit Recognizer competition.\n\n- Uses only the official `train.csv` labels.\n- Trains five CNN models with different random seeds.\n- Applies test-time augmentation with one-pixel shifts.\n- Averages logits and writes `/kaggle/working/submission.csv`.\n\nLocal submitted reference: `53258697`, public score `0.99592`.\n"}, {"cell_type": "code", "execution_count": null, "id": "e12c7fe3", "metadata": {}, "outputs": [], "source": "from pathlib import Path\nimport shutil\n\ndef find_competition_dir() -> Path:\n    for train_path in Path('/kaggle/input').rglob('train.csv'):\n        folder = train_path.parent\n        if (folder / 'test.csv').exists() and (folder / 'sample_submission.csv').exists():\n            return folder\n    raise FileNotFoundError('Could not find train.csv/test.csv/sample_submission.csv under /kaggle/input')\n\ninput_dir = find_competition_dir()\nwork_data = Path('/kaggle/working/data')\nwork_data.mkdir(parents=True, exist_ok=True)\nfor name in ['train.csv', 'test.csv', 'sample_submission.csv']:\n    shutil.copy2(input_dir / name, work_data / name)\nprint('Competition data copied from', input_dir)\n"}, {"cell_type": "code", "execution_count": null, "id": "49322837", "metadata": {}, "outputs": [], "source": "%%writefile train_official_kfold_tta.py\nfrom __future__ import annotations\n\nimport argparse\nimport hashlib\nimport json\nimport random\nfrom datetime import datetime, timezone\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn.functional as F\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch import nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\n\n\nPIXEL_COLUMNS = [f\"pixel{i}\" for i in range(784)]\nMEAN = 0.1307\nSTD = 0.3081\n\n\ndef sha256_file(path: Path) -> str:\n    digest = hashlib.sha256()\n    with path.open(\"rb\") as handle:\n        for chunk in iter(lambda: handle.read(1024 * 1024), b\"\"):\n            digest.update(chunk)\n    return digest.hexdigest()\n\n\ndef set_seed(seed: int) -> None:\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.benchmark = True\n\n\nclass MnistArrayDataset(Dataset):\n    def __init__(self, images: np.ndarray, labels: np.ndarray | None, augment: bool) -> None:\n        self.images = images.reshape(-1, 28, 28).astype(np.uint8)\n        self.labels = labels.astype(np.int64) if labels is not None else None\n        base = [\n            transforms.ToPILImage(),\n            transforms.ToTensor(),\n            transforms.Normalize((MEAN,), (STD,)),\n        ]\n        aug = [\n            transforms.ToPILImage(),\n            transforms.RandomAffine(\n                degrees=12,\n                translate=(0.10, 0.10),\n                scale=(0.90, 1.10),\n                shear=5,\n                fill=0,\n            ),\n            transforms.ToTensor(),\n            transforms.Normalize((MEAN,), (STD,)),\n            transforms.RandomErasing(p=0.15, scale=(0.02, 0.08), ratio=(0.5, 2.0), value=0),\n        ]\n        self.transform = transforms.Compose(aug if augment else base)\n\n    def __len__(self) -> int:\n        return len(self.images)\n\n    def __getitem__(self, idx: int):\n        x = self.transform(self.images[idx])\n        if self.labels is None:\n            return x\n        return x, int(self.labels[idx])\n\n\nclass ConvBlock(nn.Module):\n    def __init__(self, in_channels: int, out_channels: int) -> None:\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.SiLU(inplace=True),\n            nn.Conv2d(out_channels, out_channels, 3, padding=1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.SiLU(inplace=True),\n        )\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        return self.net(x)\n\n\nclass OfficialFoldCnn(nn.Module):\n    def __init__(self) -> None:\n        super().__init__()\n        self.features = nn.Sequential(\n            ConvBlock(1, 48),\n            nn.MaxPool2d(2),\n            nn.Dropout2d(0.10),\n            ConvBlock(48, 96),\n            nn.MaxPool2d(2),\n            nn.Dropout2d(0.15),\n            ConvBlock(96, 128),\n            nn.AdaptiveAvgPool2d((4, 4)),\n        )\n        self.classifier = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(128 * 4 * 4, 384),\n            nn.SiLU(inplace=True),\n            nn.Dropout(0.35),\n            nn.Linear(384, 10),\n        )\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        return self.classifier(self.features(x))\n\n\ndef train_one_epoch(\n    model: nn.Module,\n    loader: DataLoader,\n    optimizer: torch.optim.Optimizer,\n    scheduler: torch.optim.lr_scheduler.LRScheduler,\n    device: torch.device,\n) -> float:\n    criterion = nn.CrossEntropyLoss(label_smoothing=0.03)\n    model.train()\n    total_loss = 0.0\n    total = 0\n    for x, y in loader:\n        x = x.to(device, non_blocking=True)\n        y = y.to(device, non_blocking=True)\n        optimizer.zero_grad(set_to_none=True)\n        loss = criterion(model(x), y)\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n        total_loss += float(loss.item()) * len(y)\n        total += len(y)\n    return total_loss / total\n\n\n@torch.no_grad()\ndef predict_logits(model: nn.Module, loader: DataLoader, device: torch.device) -> torch.Tensor:\n    model.eval()\n    chunks: list[torch.Tensor] = []\n    for batch in loader:\n        if isinstance(batch, (tuple, list)):\n            x = batch[0]\n        else:\n            x = batch\n        x = x.to(device, non_blocking=True)\n        chunks.append(model(x).cpu())\n    return torch.cat(chunks, dim=0)\n\n\ndef shift_batch(x: torch.Tensor, dy: int, dx: int) -> torch.Tensor:\n    padded = F.pad(x, (1, 1, 1, 1), value=-MEAN / STD)\n    y0 = 1 + dy\n    x0 = 1 + dx\n    return padded[:, :, y0 : y0 + 28, x0 : x0 + 28]\n\n\n@torch.no_grad()\ndef predict_logits_tta(model: nn.Module, loader: DataLoader, device: torch.device) -> torch.Tensor:\n    model.eval()\n    shifts = [(0, 0), (1, 0), (-1, 0), (0, 1), (0, -1)]\n    chunks: list[torch.Tensor] = []\n    for x in loader:\n        x = x.to(device, non_blocking=True)\n        logits = torch.zeros((x.size(0), 10), device=device)\n        for dy, dx in shifts:\n            logits += model(shift_batch(x, dy, dx))\n        chunks.append((logits / len(shifts)).cpu())\n    return torch.cat(chunks, dim=0)\n\n\ndef make_loader(\n    images: np.ndarray,\n    labels: np.ndarray | None,\n    batch_size: int,\n    augment: bool,\n    shuffle: bool,\n) -> DataLoader:\n    num_workers = 2 if torch.cuda.is_available() else 0\n    return DataLoader(\n        MnistArrayDataset(images, labels, augment=augment),\n        batch_size=batch_size,\n        shuffle=shuffle,\n        num_workers=num_workers,\n        pin_memory=torch.cuda.is_available(),\n        persistent_workers=num_workers > 0,\n    )\n\n\ndef main() -> None:\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--workspace\", type=Path, default=Path.cwd())\n    parser.add_argument(\"--folds\", type=int, default=5)\n    parser.add_argument(\"--epochs\", type=int, default=12)\n    parser.add_argument(\"--batch-size\", type=int, default=512)\n    parser.add_argument(\"--seed\", type=int, default=20260601)\n    parser.add_argument(\"--tag\", default=None)\n    args = parser.parse_args()\n\n    set_seed(args.seed)\n    workspace = args.workspace.resolve()\n    tag = args.tag or f\"official_kfold{args.folds}_tta_e{args.epochs}_s{args.seed}\"\n    data_dir = workspace / \"data\"\n    outputs_dir = workspace / \"outputs\"\n    outputs_dir.mkdir(parents=True, exist_ok=True)\n\n    train_df = pd.read_csv(data_dir / \"train.csv\")\n    test_df = pd.read_csv(data_dir / \"test.csv\")\n    sample_df = pd.read_csv(data_dir / \"sample_submission.csv\")\n    x = train_df[PIXEL_COLUMNS].to_numpy(dtype=np.uint8)\n    y = train_df[\"label\"].to_numpy(dtype=np.int64)\n    test_x = test_df[PIXEL_COLUMNS].to_numpy(dtype=np.uint8)\n\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    splitter = StratifiedKFold(n_splits=args.folds, shuffle=True, random_state=args.seed)\n    test_loader = make_loader(test_x, None, args.batch_size, augment=False, shuffle=False)\n    ensemble_logits = torch.zeros((len(test_x), 10), dtype=torch.float32)\n    fold_rows: list[dict[str, object]] = []\n\n    for fold_idx, (train_idx, val_idx) in enumerate(splitter.split(x, y), start=1):\n        fold_seed = args.seed + fold_idx * 100\n        set_seed(fold_seed)\n        model = OfficialFoldCnn().to(device)\n        train_loader = make_loader(x[train_idx], y[train_idx], args.batch_size, augment=True, shuffle=True)\n        val_loader = make_loader(x[val_idx], y[val_idx], args.batch_size, augment=False, shuffle=False)\n        optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n        scheduler = torch.optim.lr_scheduler.OneCycleLR(\n            optimizer,\n            max_lr=1e-3,\n            epochs=args.epochs,\n            steps_per_epoch=len(train_loader),\n            pct_start=0.20,\n        )\n        history: list[dict[str, float]] = []\n        best_state: dict[str, torch.Tensor] | None = None\n        best_val = -1.0\n        for epoch in range(1, args.epochs + 1):\n            loss = train_one_epoch(model, train_loader, optimizer, scheduler, device)\n            val_logits = predict_logits(model, val_loader, device)\n            val_preds = val_logits.argmax(dim=1).numpy()\n            val_acc = float((val_preds == y[val_idx]).mean())\n            row = {\"fold\": fold_idx, \"epoch\": epoch, \"loss\": loss, \"val_accuracy\": val_acc}\n            history.append(row)\n            print(json.dumps(row), flush=True)\n            if val_acc > best_val:\n                best_val = val_acc\n                best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}\n\n        if best_state is not None:\n            model.load_state_dict(best_state)\n        fold_test_logits = predict_logits_tta(model, test_loader, device)\n        ensemble_logits += fold_test_logits\n        fold_rows.append(\n            {\n                \"fold\": fold_idx,\n                \"seed\": fold_seed,\n                \"train_rows\": int(len(train_idx)),\n                \"validation_rows\": int(len(val_idx)),\n                \"best_val_accuracy\": best_val,\n                \"history\": history,\n            }\n        )\n\n    preds = ensemble_logits.argmax(dim=1).numpy().astype(np.int64)\n    submission = sample_df.copy()\n    submission[\"Label\"] = preds\n    submission_path = outputs_dir / f\"submission_{tag}.csv\"\n    metrics_path = outputs_dir / f\"submission_{tag}.json\"\n    submission.to_csv(submission_path, index=False)\n\n    metrics = {\n        \"created_utc\": datetime.now(timezone.utc).isoformat(),\n        \"method\": \"official_train_only_kfold_cnn_tta\",\n        \"folds\": args.folds,\n        \"epochs\": args.epochs,\n        \"seed\": args.seed,\n        \"device\": str(device),\n        \"train_rows\": int(len(x)),\n        \"test_rows\": int(len(test_x)),\n        \"folds_detail\": fold_rows,\n        \"mean_best_val_accuracy\": float(np.mean([r[\"best_val_accuracy\"] for r in fold_rows])),\n        \"submission_path\": str(submission_path),\n        \"submission_sha256\": sha256_file(submission_path),\n    }\n    metrics_path.write_text(json.dumps(metrics, indent=2), encoding=\"utf-8\")\n    print(json.dumps(metrics, indent=2), flush=True)\n\n\nif __name__ == \"__main__\":\n    main()\n"}, {"cell_type": "code", "execution_count": null, "id": "28121477", "metadata": {}, "outputs": [], "source": "%%writefile train_official_seed_ensemble_tta.py\nfrom __future__ import annotations\n\nimport argparse\nimport hashlib\nimport json\nfrom datetime import datetime, timezone\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\n\nfrom train_official_kfold_tta import (\n    OfficialFoldCnn,\n    PIXEL_COLUMNS,\n    make_loader,\n    predict_logits_tta,\n    set_seed,\n    train_one_epoch,\n)\n\n\ndef sha256_file(path: Path) -> str:\n    digest = hashlib.sha256()\n    with path.open(\"rb\") as handle:\n        for chunk in iter(lambda: handle.read(1024 * 1024), b\"\"):\n            digest.update(chunk)\n    return digest.hexdigest()\n\n\ndef main() -> None:\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--workspace\", type=Path, default=Path.cwd())\n    parser.add_argument(\"--models\", type=int, default=5)\n    parser.add_argument(\"--epochs\", type=int, default=16)\n    parser.add_argument(\"--batch-size\", type=int, default=512)\n    parser.add_argument(\"--seed\", type=int, default=20260601)\n    parser.add_argument(\"--tag\", default=None)\n    args = parser.parse_args()\n\n    workspace = args.workspace.resolve()\n    tag = args.tag or f\"official_seed{args.models}_tta_e{args.epochs}_s{args.seed}\"\n    data_dir = workspace / \"data\"\n    outputs_dir = workspace / \"outputs\"\n    outputs_dir.mkdir(parents=True, exist_ok=True)\n\n    train_df = pd.read_csv(data_dir / \"train.csv\")\n    test_df = pd.read_csv(data_dir / \"test.csv\")\n    sample_df = pd.read_csv(data_dir / \"sample_submission.csv\")\n    x = train_df[PIXEL_COLUMNS].to_numpy(dtype=np.uint8)\n    y = train_df[\"label\"].to_numpy(dtype=np.int64)\n    test_x = test_df[PIXEL_COLUMNS].to_numpy(dtype=np.uint8)\n\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    train_loader = make_loader(x, y, args.batch_size, augment=True, shuffle=True)\n    test_loader = make_loader(test_x, None, args.batch_size, augment=False, shuffle=False)\n    ensemble_logits = torch.zeros((len(test_x), 10), dtype=torch.float32)\n    model_rows: list[dict[str, object]] = []\n\n    for model_idx in range(1, args.models + 1):\n        model_seed = args.seed + model_idx * 1000\n        set_seed(model_seed)\n        model = OfficialFoldCnn().to(device)\n        optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n        scheduler = torch.optim.lr_scheduler.OneCycleLR(\n            optimizer,\n            max_lr=1e-3,\n            epochs=args.epochs,\n            steps_per_epoch=len(train_loader),\n            pct_start=0.20,\n        )\n        history: list[dict[str, float]] = []\n        for epoch in range(1, args.epochs + 1):\n            loss = train_one_epoch(model, train_loader, optimizer, scheduler, device)\n            row = {\"model\": model_idx, \"epoch\": epoch, \"loss\": loss}\n            history.append(row)\n            print(json.dumps(row), flush=True)\n        ensemble_logits += predict_logits_tta(model, test_loader, device)\n        model_rows.append({\"model\": model_idx, \"seed\": model_seed, \"history\": history})\n\n    preds = ensemble_logits.argmax(dim=1).numpy().astype(np.int64)\n    submission = sample_df.copy()\n    submission[\"Label\"] = preds\n    submission_path = outputs_dir / f\"submission_{tag}.csv\"\n    metrics_path = outputs_dir / f\"submission_{tag}.json\"\n    submission.to_csv(submission_path, index=False)\n\n    metrics = {\n        \"created_utc\": datetime.now(timezone.utc).isoformat(),\n        \"method\": \"official_train_only_seed_ensemble_cnn_tta\",\n        \"models\": args.models,\n        \"epochs\": args.epochs,\n        \"seed\": args.seed,\n        \"device\": str(device),\n        \"train_rows\": int(len(x)),\n        \"test_rows\": int(len(test_x)),\n        \"models_detail\": model_rows,\n        \"submission_path\": str(submission_path),\n        \"submission_sha256\": sha256_file(submission_path),\n    }\n    metrics_path.write_text(json.dumps(metrics, indent=2), encoding=\"utf-8\")\n    print(json.dumps(metrics, indent=2), flush=True)\n\n\nif __name__ == \"__main__\":\n    main()\n"}, {"cell_type": "code", "execution_count": null, "id": "a83ee1d4", "metadata": {}, "outputs": [], "source": "!python train_official_seed_ensemble_tta.py --workspace /kaggle/working --models 5 --epochs 16 --batch-size 512 --seed 20260601 --tag official5seed_tta_e16_s20260601\n\nsubmission_src = Path('/kaggle/working/outputs/submission_official5seed_tta_e16_s20260601.csv')\nsubmission_dst = Path('/kaggle/working/submission.csv')\nshutil.copy2(submission_src, submission_dst)\nprint('Wrote', submission_dst)\n"}, {"cell_type": "code", "execution_count": null, "id": "cb49894f", "metadata": {}, "outputs": [], "source": "import json\nimport pandas as pd\n\nsubmission = pd.read_csv('/kaggle/working/submission.csv')\nmetrics = json.loads(Path('/kaggle/working/outputs/submission_official5seed_tta_e16_s20260601.json').read_text())\nprint(submission.shape)\nprint(submission.head())\nprint(json.dumps({\n    'method': metrics['method'],\n    'models': metrics['models'],\n    'epochs': metrics['epochs'],\n    'sha256': metrics['submission_sha256'],\n}, indent=2))\n"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "pygments_lexer": "ipython3"}}, "nbformat": 4, "nbformat_minor": 5}