{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ======================================================\n# Cell 1: Train a plain custom ConvNet on CIFAR-10\n# - Random init (no pretrained)\n# - No rotation (only crop + horizontal flip)\n# - Saves best weights to: best_convnet.pt\n# - Trains on official CIFAR-10 via torchvision (fast; no need to extract train.7z)\n# ======================================================\nimport os, random, warnings\nwarnings.filterwarnings(\"ignore\")\nimport numpy as np\nimport torch, torch.nn as nn, torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Subset\nfrom torchvision import datasets, transforms\nfrom torch.cuda.amp import autocast, GradScaler\n\n# Repro + device\nSEED=1337\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ntorch.backends.cudnn.benchmark = True\nprint(\"Device:\", device)\n\n# CIFAR-10 stats and transforms (NO rotation)\nCIFAR_MEAN=(0.4914,0.4822,0.4465)\nCIFAR_STD =(0.2470,0.2435,0.2616)\ntrain_tfms = transforms.Compose([\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomHorizontalFlip(),  # ✅ no rotation\n    transforms.ToTensor(),\n    transforms.Normalize(CIFAR_MEAN, CIFAR_STD),\n    transforms.RandomErasing(p=0.25),\n])\ntest_tfms  = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(CIFAR_MEAN, CIFAR_STD),\n])\n\n# Load official CIFAR-10 (download once to /kaggle/working)\nDATA_ROOT=\"/kaggle/working\"\n_base = datasets.CIFAR10(root=DATA_ROOT, train=True, download=True, transform=None)\nidx = np.arange(len(_base)); np.random.default_rng(SEED).shuffle(idx)\nVAL_SZ=5000\ntrain_idx, val_idx = idx[VAL_SZ:], idx[:VAL_SZ]\n\ntrain_set = Subset(datasets.CIFAR10(root=DATA_ROOT, train=True, download=False, transform=train_tfms), train_idx.tolist())\nval_set   = Subset(datasets.CIFAR10(root=DATA_ROOT, train=True, download=False, transform=test_tfms),  val_idx.tolist())\n\nBATCH_TRAIN, BATCH_VAL = 256, 512\ntrain_loader = DataLoader(train_set, batch_size=BATCH_TRAIN, shuffle=True,  num_workers=2, pin_memory=True)\nval_loader   = DataLoader(val_set,   batch_size=BATCH_VAL,   shuffle=False, num_workers=2, pin_memory=True)\n\nprint(f\"Train rows: {len(train_set)} | Val rows: {len(val_set)}\")\n\n# Model: plain custom ConvNet (random init)\nclass Net(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.c1=nn.Conv2d(3,64,3,padding=1);  self.b1=nn.BatchNorm2d(64)\n        self.c2=nn.Conv2d(64,64,3,padding=1); self.b2=nn.BatchNorm2d(64)\n        self.p1=nn.MaxPool2d(2,2)\n        self.c3=nn.Conv2d(64,128,3,padding=1); self.b3=nn.BatchNorm2d(128)\n        self.c4=nn.Conv2d(128,128,3,padding=1);self.b4=nn.BatchNorm2d(128)\n        self.p2=nn.MaxPool2d(2,2)\n        self.c5=nn.Conv2d(128,256,3,padding=1);self.b5=nn.BatchNorm2d(256)\n        self.c6=nn.Conv2d(256,256,3,padding=1);self.b6=nn.BatchNorm2d(256)\n        self.p3=nn.MaxPool2d(2,2)\n        self.fc1=nn.Linear(256*4*4,512); self.drop=nn.Dropout(0.5)\n        self.fc2=nn.Linear(512,10)\n    def forward(self,x):\n        x=F.relu(self.b1(self.c1(x))); x=F.relu(self.b2(self.c2(x))); x=self.p1(x)\n        x=F.relu(self.b3(self.c3(x))); x=F.relu(self.b4(self.c4(x))); x=self.p2(x)\n        x=F.relu(self.b5(self.c5(x))); x=F.relu(self.b6(self.c6(x))); x=self.p3(x)\n        x=torch.flatten(x,1); x=F.relu(self.fc1(x)); x=self.drop(x)\n        return self.fc2(x)\n\nmodel = Net().to(device).to(memory_format=torch.channels_last)\n\n# Optimizer / Scheduler / Loss / AMP\nEPOCHS=60\nopt = torch.optim.SGD(model.parameters(), lr=0.05, momentum=0.9, weight_decay=5e-4, nesterov=True)\nsch = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=EPOCHS)\ncrit = nn.CrossEntropyLoss(label_smoothing=0.05)\nscaler = GradScaler()\n\ndef run_epoch(dl, train=True):\n    model.train(train)\n    tot=correct=loss_sum=0.0\n    for x,y in dl:\n        x=x.to(device,non_blocking=True).contiguous(memory_format=torch.channels_last)\n        y=y.to(device,non_blocking=True)\n        if train: opt.zero_grad(set_to_none=True)\n        with autocast():\n            logits=model(x); loss=crit(logits,y)\n        if train:\n            scaler.scale(loss).backward(); scaler.step(opt); scaler.update()\n        loss_sum+=loss.item()*x.size(0); correct+=(logits.argmax(1)==y).sum().item(); tot+=x.size(0)\n    return loss_sum/tot, correct/tot\n\nbest=(0.0,None)\nfor e in range(1, EPOCHS+1):\n    tr_loss,tr_acc = run_epoch(train_loader, True)\n    va_loss,va_acc = run_epoch(val_loader,   False)\n    sch.step()\n    if va_acc>best[0]: best=(va_acc, {k:v.detach().cpu() for k,v in model.state_dict().items()})\n    print(f\"Ep{e:02d}/{EPOCHS}  train_acc={tr_acc:.3f}  val_acc={va_acc:.3f}\")\n\n# Save best weights\ntorch.save(best[1], \"best_convnet.pt\")\nprint(\"✅ Saved best_convnet.pt\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-24T17:13:01.657797Z","iopub.execute_input":"2025-09-24T17:13:01.658003Z","iopub.status.idle":"2025-09-24T17:26:24.968133Z","shell.execute_reply.started":"2025-09-24T17:13:01.657983Z","shell.execute_reply":"2025-09-24T17:26:24.966886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!apt install libarchive-dev\n\n!pip install libarchive\n\nimport libarchive.public\n\ncnt = 0\n\nfor entry in libarchive.public.file_pour('/kaggle/input/cifar-10/test.7z'):\n    cnt += 1\n    if cnt % 1000 == 0: print(cnt)\n\n \n\nimport glob\nlen(glob.glob('test/*'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T17:37:20.921750Z","iopub.execute_input":"2025-09-24T17:37:20.922040Z","iopub.status.idle":"2025-09-24T17:38:43.811802Z","shell.execute_reply.started":"2025-09-24T17:37:20.922019Z","shell.execute_reply":"2025-09-24T17:38:43.810853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================================\n# Cell 2: Extraction with OVERALL COUNT (e.g., 12345/300000)\n# - Extracts /kaggle/input/cifar-10/test.7z → ./test/\n# - Prints one updating line: \"<done>/<total> (<pct>%)\"\n# - Skips if already extracted (≥300k)\n# ======================================================\nimport os, glob, shutil, subprocess, sys, time\nimport os\nimport libarchive.public\nimport glob\n\n# Define dataset base directory\nCOMP_DIR = \"/kaggle/input/cifar-10\"\nTEST_7Z = os.path.join(COMP_DIR, \"test.7z\")\nEXTRACT_DIR = \"test\"\n\n# Extract using libarchive (⚠️ very slow for large .7z files)\ncnt = 0\nprint(\"⏳ Reading entries from test.7z...\")\n\nfor entry in libarchive.public.file_pour(TEST_7Z):\n    cnt += 1\n    if cnt % 1000 == 0:\n        print(cnt)\n\nprint(f\"✅ Total entries read: {cnt}\")\n\n# Optional: Count number of extracted PNG files\npng_count = len(glob.glob(os.path.join(EXTRACT_DIR, \"*.png\")))\nprint(f\"🧾 Extracted image count in '{EXTRACT_DIR}/': {png_count}\")\n\nTEST_7Z  = os.path.join(COMP_DIR, \"test.7z\")\nTEST_DIR = \"./test\"\nTOTAL    = 300_000  # CIFAR-10 Kaggle test count\n\nassert os.path.exists(TEST_7Z), \"Missing /kaggle/input/cifar-10/test.7z\"\nos.makedirs(TEST_DIR, exist_ok=True)\n\ndef ensure_7z():\n    if shutil.which(\"7z\"):\n        return \"7z\"\n    subprocess.run([\"apt-get\",\"update\",\"-y\"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n    subprocess.run([\"apt-get\",\"install\",\"-y\",\"p7zip-full\"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n    assert shutil.which(\"7z\"), \"7z not found after install\"\n    return \"7z\"\n\n# If already extracted, skip\nhave = len(glob.glob(os.path.join(TEST_DIR, \"*.png\")))\nif have >= TOTAL:\n    print(f\"✅ Already extracted: {have} / {TOTAL} in {TEST_DIR}\")\nelse:\n    sevenz = ensure_7z()\n    print(f\"Extracting to {os.path.abspath(TEST_DIR)}\")\n    # Start extraction in the background\n    # Archive has 'test/...' inside; extract to current dir so files land in ./test/\n    proc = subprocess.Popen(\n        [sevenz, \"x\", TEST_7Z, \"-y\", \"-o.\"],\n        stdout=subprocess.DEVNULL,\n        stderr=subprocess.DEVNULL\n    )\n\n    # Poll progress by counting files in ./test/*.png\n    last_print = -1\n    try:\n        while True:\n            # count extracted PNGs so far\n            done = len(glob.glob(os.path.join(TEST_DIR, \"*.png\")))\n            if done != last_print:\n                pct = 100.0 * min(done, TOTAL) / TOTAL\n                print(f\"\\rprogress: {done}/{TOTAL} ({pct:5.1f}%)\", end=\"\", flush=True)\n                last_print = done\n            # break when process finishes and count stops increasing\n            if proc.poll() is not None:\n                # one last update after process ends\n                done = len(glob.glob(os.path.join(TEST_DIR, \"*.png\")))\n                pct = 100.0 * min(done, TOTAL) / TOTAL\n                print(f\"\\rprogress: {done}/{TOTAL} ({pct:5.1f}%)\")\n                break\n            time.sleep(1)\n    finally:\n        # Ensure process is not left hanging\n        try:\n            proc.terminate()\n        except Exception:\n            pass\n\n    final = len(glob.glob(os.path.join(TEST_DIR, \"*.png\")))\n    print(f\"✅ Extracted: {final} / {TOTAL} in {TEST_DIR}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T17:44:59.561588Z","iopub.execute_input":"2025-09-24T17:44:59.562167Z","iopub.status.idle":"2025-09-24T17:46:20.212263Z","shell.execute_reply.started":"2025-09-24T17:44:59.562135Z","shell.execute_reply":"2025-09-24T17:46:20.211369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================================\n# Cell 3: Inference on Kaggle test set\n# - Loads best_convnet.pt\n# - If ./test/ exists: read PNGs directly\n# - Else: batched tiny-extract from test.7z with py7zr (no full unzip)\n# - Saves ids.npy and pred_idx.npy for the submission cell\n# ======================================================\nimport os, glob, tempfile, subprocess, numpy as np, pandas as pd\nfrom PIL import Image\nimport torch\nfrom torchvision import transforms\n\n# Load model\nCLASS_NAMES=['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']\nfrom torch import nn\nclass Net(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.c1=nn.Conv2d(3,64,3,padding=1);  self.b1=nn.BatchNorm2d(64)\n        self.c2=nn.Conv2d(64,64,3,padding=1); self.b2=nn.BatchNorm2d(64)\n        self.p1=nn.MaxPool2d(2,2)\n        self.c3=nn.Conv2d(64,128,3,padding=1); self.b3=nn.BatchNorm2d(128)\n        self.c4=nn.Conv2d(128,128,3,padding=1);self.b4=nn.BatchNorm2d(128)\n        self.p2=nn.MaxPool2d(2,2)\n        self.c5=nn.Conv2d(128,256,3,padding=1);self.b5=nn.BatchNorm2d(256)\n        self.c6=nn.Conv2d(256,256,3,padding=1);self.b6=nn.BatchNorm2d(256)\n        self.p3=nn.MaxPool2d(2,2)\n        self.fc1=nn.Linear(256*4*4,512); self.drop=nn.Dropout(0.5)\n        self.fc2=nn.Linear(512,10)\n    def forward(self,x):\n        import torch.nn.functional as F\n        x=F.relu(self.b1(self.c1(x))); x=F.relu(self.b2(self.c2(x))); x=self.p1(x)\n        x=F.relu(self.b3(self.c3(x))); x=F.relu(self.b4(self.c4(x))); x=self.p2(x)\n        x=F.relu(self.b5(self.c5(x))); x=F.relu(self.b6(self.c6(x))); x=self.p3(x)\n        x=torch.flatten(x,1); x=F.relu(self.fc1(x)); x=self.drop(x)\n        return self.fc2(x)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = Net().to(device).to(memory_format=torch.channels_last)\nstate = torch.load(\"best_convnet.pt\", map_location=\"cpu\")\nmodel.load_state_dict(state)\nmodel.eval()\nprint(\"✅ Loaded best_convnet.pt\")\n\n# Paths and transforms\nCOMP_DIR = \"/kaggle/input/cifar-10\"\nSAMPLE_SUB = os.path.join(COMP_DIR, \"sampleSubmission.csv\")\nTEST_7Z    = os.path.join(COMP_DIR, \"test.7z\")\nTEST_DIR   = \"./test\"\nassert os.path.exists(SAMPLE_SUB), \"Missing sampleSubmission.csv\"\n\nCIFAR_MEAN=(0.4914,0.4822,0.4465); CIFAR_STD=(0.2470,0.2435,0.2616)\neval_tfm = transforms.Compose([transforms.ToTensor(), transforms.Normalize(CIFAR_MEAN, CIFAR_STD)])\n\n# Read ID order\nsample = pd.read_csv(SAMPLE_SUB)\nids = sample[\"id\"].astype(int).tolist()\nN = len(ids)\nprint(\"IDs to predict:\", N)\n\nfrom torch.cuda.amp import autocast\nimport torch\n\nBATCH = 1024  # lower if OOM\n\npred_idx = np.empty(N, dtype=np.int64)\n\nif os.path.isdir(TEST_DIR) and glob.glob(os.path.join(TEST_DIR, \"1.png\")):\n    # -------- Path A: fully extracted test/ exists --------\n    print(\"Using extracted files in ./test/\")\n    def batched(iterable, n):\n        for i in range(0, len(iterable), n):\n            yield i, iterable[i:i+n]\n    with torch.no_grad():\n        done=0\n        for start, id_batch in batched(ids, BATCH):\n            imgs=[]\n            for i in id_batch:\n                p = os.path.join(TEST_DIR, f\"{i}.png\")\n                img = Image.open(p).convert(\"RGB\")\n                imgs.append(eval_tfm(img))\n            xb = torch.stack(imgs).to(device, non_blocking=True).contiguous(memory_format=torch.channels_last)\n            with autocast():\n                logits = model(xb)\n            preds = logits.argmax(1).cpu().numpy()\n            pred_idx[start:start+len(id_batch)] = preds\n            done += len(id_batch)\n            if done % 10000 == 0: print(f\"progress: {done}/{N}\")\nelse:\n    # -------- Path B: no extraction → tiny batched extracts from test.7z --------\n    print(\"No ./test/ found; using batched tiny-extract from test.7z\")\n    try:\n        import py7zr  # noqa\n    except Exception:\n        subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"py7zr\"], check=True)\n        import py7zr\n\n    def batched(iterable, n):\n        for i in range(0, len(iterable), n):\n            yield i, iterable[i:i+n]\n\n    with torch.no_grad(), py7zr.SevenZipFile(TEST_7Z, mode=\"r\") as z:\n        done=0\n        for start, id_batch in batched(ids, BATCH):\n            with tempfile.TemporaryDirectory(dir=\"/kaggle/temp\") as tmpdir:\n                targets = [f\"{i}.png\" for i in id_batch]\n                # extract ONLY this batch to tmpdir\n                try:\n                    z.extract(path=tmpdir, targets=targets)\n                except TypeError:\n                    z.extract(targets=targets, path=tmpdir)\n                imgs=[]\n                for i in id_batch:\n                    p = os.path.join(tmpdir, f\"{i}.png\")\n                    img = Image.open(p).convert(\"RGB\")\n                    imgs.append(eval_tfm(img))\n                xb = torch.stack(imgs).to(device, non_blocking=True).contiguous(memory_format=torch.channels_last)\n                with autocast():\n                    logits = model(xb)\n                preds = logits.argmax(1).cpu().numpy()\n                pred_idx[start:start+len(id_batch)] = preds\n            done += len(id_batch)\n            if done % 10000 == 0: print(f\"progress: {done}/{N}\")\n\n# Save inference artifacts for the submission cell\nnp.save(\"ids.npy\", np.array(ids, dtype=np.int32))\nnp.save(\"pred_idx.npy\", pred_idx)\nprint(\"✅ Saved ids.npy and pred_idx.npy\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T17:46:40.031573Z","iopub.execute_input":"2025-09-24T17:46:40.031869Z","iopub.status.idle":"2025-09-24T17:49:11.338730Z","shell.execute_reply.started":"2025-09-24T17:46:40.031850Z","shell.execute_reply":"2025-09-24T17:49:11.337781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================================================\n# Cell 4: Build submission.csv\n# - Loads ids.npy + pred_idx.npy from Cell 3\n# - Maps class indices → labels\n# - Writes submission.csv\n# ======================================================\nimport numpy as np, pandas as pd\n\nCLASS_NAMES=['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']\n\nids = np.load(\"ids.npy\")\npred_idx = np.load(\"pred_idx.npy\")\nlabels = [CLASS_NAMES[i] for i in pred_idx.tolist()]\n\nsub = pd.DataFrame({\"id\": ids.astype(int), \"label\": labels})\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"✅ Wrote submission.csv\")\nprint(sub.head())\nprint(\"shape:\", sub.shape, \"| columns:\", list(sub.columns), \"| ids unique:\", sub['id'].is_unique)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T17:49:22.320446Z","iopub.execute_input":"2025-09-24T17:49:22.320765Z","iopub.status.idle":"2025-09-24T17:49:22.609525Z","shell.execute_reply.started":"2025-09-24T17:49:22.320741Z","shell.execute_reply":"2025-09-24T17:49:22.608542Z"}},"outputs":[],"execution_count":null}]}