{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"After reading the following papers: \\\n1.\thttps://arxiv.org/html/2501.06220v1 \\\n2.\thttps://app.readytensor.ai/publications/vision-transformer-vit-for-image-classification-on-cifar- 10-XcqsPj3d18JS\n3.\thttps://dmicz.github.io/machine-learning/minvit\n\nI decided to use vision transformer, but I have not much strong hardware, I decided to stop on choosing the tiny ViT for model as the paper suggest it will show strong results. For hyperparameter optimization I tried to not take computationally cost technique and not weak one and stopped on \"Population-Based Training\" hyperparameter optimization. With the strict layer = 1, my hypothesis that it will not show impressive results, we will see. Let's start our implementation\n\nInitially I decided to use Population-Based Training for hyperparameter optimization after reading this paper:\n1. https://www.thelancet.com/journals/ebiom/article/PIIS2352-3964(25)00101-X/fulltext\n2. https://arxiv.org/abs/2504.10231\n\nBut due to time constraints and hardware resources I decided to apply random search because it is more efficient than grid search in high-dimensional spaces, especially when some hyperparameters are more important than others ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"### Imports and setups","metadata":{}},{"cell_type":"code","source":"# Keep in mind that I'm using CIFAR - 10 dataset, for model I'm using TinyViT + Random Search\nimport os\nimport time\nimport glob\nimport math\nimport json\nimport random\nfrom pathlib import Path\nfrom functools import partial\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nfrom torchvision import transforms as T\nfrom torch.utils.data import DataLoader, random_split\nprint(\"Setup was done\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:29.078710Z","iopub.execute_input":"2025-11-14T16:58:29.079422Z","iopub.status.idle":"2025-11-14T16:58:36.263829Z","shell.execute_reply.started":"2025-11-14T16:58:29.079380Z","shell.execute_reply":"2025-11-14T16:58:36.263208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# For my own convenience I'm using reproducability to not see everytime different results\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nset_seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:43.005888Z","iopub.execute_input":"2025-11-14T16:58:43.006337Z","iopub.status.idle":"2025-11-14T16:58:43.018092Z","shell.execute_reply.started":"2025-11-14T16:58:43.006314Z","shell.execute_reply":"2025-11-14T16:58:43.017314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Checking the device, whether I'm using the GPU or not","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:44.609121Z","iopub.execute_input":"2025-11-14T16:58:44.609704Z","iopub.status.idle":"2025-11-14T16:58:44.676206Z","shell.execute_reply.started":"2025-11-14T16:58:44.609676Z","shell.execute_reply":"2025-11-14T16:58:44.675422Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Data preparation and preprocessing","metadata":{}},{"cell_type":"markdown","source":"I'm doing the following 2 lines of codes to normalize my images","metadata":{}},{"cell_type":"code","source":"cifar10_mean = (0.4914, 0.4822, 0.4465)\ncifar10_std  = (0.2470, 0.2435, 0.2616)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:45.890532Z","iopub.execute_input":"2025-11-14T16:58:45.890810Z","iopub.status.idle":"2025-11-14T16:58:45.894687Z","shell.execute_reply.started":"2025-11-14T16:58:45.890787Z","shell.execute_reply":"2025-11-14T16:58:45.894103Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"I applied Random Augmentation, it is a powerful augmentation from Google Research","metadata":{}},{"cell_type":"code","source":"def get_transforms(use_randaugment=True, cutout=False):\n    train_ops = [\n        T.RandomResizedCrop(32, scale=(0.8, 1.0)),\n        T.RandomHorizontalFlip(),\n        T.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n    ]\n    if use_randaugment and hasattr(T, \"RandAugment\"):\n        train_ops.append(T.RandAugment(num_ops=2, magnitude=9))\n    train_ops.extend([T.ToTensor(), T.Normalize(cifar10_mean, cifar10_std)])\n    if cutout:\n        \n        class Cutout:\n            def __init__(self, n_holes=1, length=8):\n                self.n_holes = n_holes\n                self.length = length\n            def __call__(self, img):\n                c,h,w = img.shape\n                mask = np.ones((h,w), np.float32)\n                for _ in range(self.n_holes):\n                    y = np.random.randint(h)\n                    x = np.random.randint(w)\n                    y1 = np.clip(y-self.length//2,0,h)\n                    y2 = np.clip(y+self.length//2,0,h)\n                    x1 = np.clip(x-self.length//2,0,w)\n                    x2 = np.clip(x+self.length//2,0,w)\n                    mask[y1:y2, x1:x2] = 0.\n                mask = torch.from_numpy(mask).type_as(img[0])\n                mask = mask.unsqueeze(0).repeat(c,1,1)\n                return img*mask\n        train_ops.append(Cutout())\n    train_transform = T.Compose(train_ops)\n    test_transform = T.Compose([\n        T.Resize(32),\n        T.ToTensor(),\n        T.Normalize(cifar10_mean, cifar10_std)\n    ])\n    return train_transform, test_transform","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:48.341352Z","iopub.execute_input":"2025-11-14T16:58:48.342071Z","iopub.status.idle":"2025-11-14T16:58:48.350090Z","shell.execute_reply.started":"2025-11-14T16:58:48.342045Z","shell.execute_reply":"2025-11-14T16:58:48.349203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_cifar10_loaders_with_val(batch_size=128, data_dir=\"./data\",\n                                 use_randaugment=True, cutout=False,\n                                 val_fraction=0.1, seed=42, num_workers=4):\n    train_transform, test_transform = get_transforms(use_randaugment, cutout)\n    full_train = torchvision.datasets.CIFAR10(root=data_dir, train=True, download=True, transform=train_transform)\n    test_set = torchvision.datasets.CIFAR10(root=data_dir, train=False, download=True, transform=test_transform)\n\n    if val_fraction > 0.0:\n        n = len(full_train)\n        val_n = int(n * val_fraction)\n        train_n = n - val_n\n        train_set, val_set = random_split(full_train, [train_n, val_n], generator=torch.Generator().manual_seed(seed))\n        val_indices = val_set.indices if hasattr(val_set, \"indices\") else None\n        if val_indices is not None:\n            val_set = torchvision.datasets.CIFAR10(root=data_dir, train=True, download=False, transform=test_transform)\n            from torch.utils.data import Subset\n            val_set = Subset(val_set, val_indices)\n        train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True)\n        val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)\n    else:\n        train_loader = DataLoader(full_train, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True)\n        val_loader = None\n\n    test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True)\n    return train_loader, val_loader, test_loader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:50.624118Z","iopub.execute_input":"2025-11-14T16:58:50.624414Z","iopub.status.idle":"2025-11-14T16:58:50.631819Z","shell.execute_reply.started":"2025-11-14T16:58:50.624391Z","shell.execute_reply":"2025-11-14T16:58:50.631193Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### TinyViT 1-layer Model","metadata":{}},{"cell_type":"code","source":"class TinyViT(nn.Module):\n    def __init__(self, img_size=32, patch_size=4, in_ch=3, emb_dim=128, mlp_dim=256, heads=4, dropout=0.1, num_classes=10):\n        super().__init__()\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.in_ch = in_ch\n        self.emb_dim = emb_dim\n        self.num_patches = (img_size // patch_size)**2\n        self.patch_dim = in_ch * patch_size * patch_size\n\n        self.proj = nn.Linear(self.patch_dim, emb_dim)\n        self.cls_token = nn.Parameter(torch.zeros(1,1,emb_dim))\n        self.pos_embed = nn.Parameter(torch.zeros(1,self.num_patches+1,emb_dim))\n\n        encoder_layer = nn.TransformerEncoderLayer(d_model=emb_dim, nhead=heads, dim_feedforward=mlp_dim,\n                                                   dropout=dropout, batch_first=True)\n        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=1)\n        self.norm = nn.LayerNorm(emb_dim)\n        self.head = nn.Linear(emb_dim, num_classes)\n\n        nn.init.trunc_normal_(self.pos_embed, std=0.02)\n        nn.init.trunc_normal_(self.cls_token, std=0.02)\n        nn.init.normal_(self.head.weight, std=0.02)\n\n    def _patchify(self, x):\n        B,C,H,W = x.shape\n        p = self.patch_size\n        x = x.view(B,C,H//p,p,W//p,p)\n        x = x.permute(0,2,4,1,3,5).contiguous()\n        x = x.view(B, -1, C*p*p)\n        return x\n\n    def forward(self, x):\n        B = x.size(0)\n        x = self._patchify(x)\n        x = self.proj(x)\n        cls_tokens = self.cls_token.expand(B,-1,-1)\n        x = torch.cat([cls_tokens, x], dim=1)\n        x = x + self.pos_embed\n        x = self.encoder(x)\n        x = self.norm(x[:,0])\n        x = self.head(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:51.896529Z","iopub.execute_input":"2025-11-14T16:58:51.896821Z","iopub.status.idle":"2025-11-14T16:58:51.905685Z","shell.execute_reply.started":"2025-11-14T16:58:51.896797Z","shell.execute_reply":"2025-11-14T16:58:51.904875Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Training Utilities","metadata":{}},{"cell_type":"code","source":"def mixup_data(x,y,alpha=0.2):\n    if alpha <= 0: return x,y,None,1.0\n    lam = np.random.beta(alpha,alpha)\n    index = torch.randperm(x.size(0)).to(x.device)\n    mixed_x = lam*x + (1-lam)*x[index,:]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:56.591562Z","iopub.execute_input":"2025-11-14T16:58:56.591847Z","iopub.status.idle":"2025-11-14T16:58:56.596602Z","shell.execute_reply.started":"2025-11-14T16:58:56.591822Z","shell.execute_reply":"2025-11-14T16:58:56.596006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_epoch(model, loader, criterion, optimizer, device, use_mixup=False, mixup_alpha=0.2):\n    model.train()\n    running_loss, correct, total = 0.0,0,0\n    t0 = time.time()\n    for xb,yb in loader:\n        xb,yb = xb.to(device), yb.to(device)\n        if use_mixup and mixup_alpha>0:\n            xb, y_a, y_b, lam = mixup_data(xb,yb,mixup_alpha)\n            out = model(xb)\n            loss = lam*criterion(out,y_a) + (1-lam)*criterion(out,y_b)\n        else:\n            out = model(xb)\n            loss = criterion(out,yb)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()*xb.size(0)\n        preds = out.argmax(dim=1)\n        if use_mixup and mixup_alpha>0:\n            correct += (preds==y_a).sum().item()\n        else:\n            correct += (preds==yb).sum().item()\n        total += xb.size(0)\n    return running_loss/total, correct/total, time.time()-t0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:58.068994Z","iopub.execute_input":"2025-11-14T16:58:58.069287Z","iopub.status.idle":"2025-11-14T16:58:58.075749Z","shell.execute_reply.started":"2025-11-14T16:58:58.069266Z","shell.execute_reply":"2025-11-14T16:58:58.074987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate(model, loader, criterion, device):\n    model.eval()\n    running_loss, correct, total = 0.0,0,0\n    t0=time.time()\n    with torch.no_grad():\n        for xb,yb in loader:\n            xb,yb = xb.to(device), yb.to(device)\n            out = model(xb)\n            loss = criterion(out,yb)\n            running_loss += loss.item()*xb.size(0)\n            preds = out.argmax(dim=1)\n            correct += (preds==yb).sum().item()\n            total += xb.size(0)\n    return running_loss/total, correct/total, time.time()-t0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T16:58:59.022139Z","iopub.execute_input":"2025-11-14T16:58:59.022743Z","iopub.status.idle":"2025-11-14T16:58:59.027784Z","shell.execute_reply.started":"2025-11-14T16:58:59.022722Z","shell.execute_reply":"2025-11-14T16:58:59.027061Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Baseline TinyViT (1 layer) run","metadata":{}},{"cell_type":"markdown","source":"Here by \"baseline\" I mean the model that is not applied any hyperparameter optimizations. I decided to train the baseline for 20 epochs, but the hyperparameter opimization will be applied more than that","metadata":{}},{"cell_type":"code","source":"baseline_config = {\n    \"patch_size\": 4,\n    \"emb_dim\": 64,\n    \"mlp_mult\": 2,\n    \"heads\": 4,\n    \"dropout\": 0.1,\n    \"lr\": 3e-4,\n    \"weight_decay\": 0.05,\n    \"batch_size\": 128,\n    \"mixup_alpha\": 0.0,\n    \"label_smoothing\": 0.0,\n    \"randaugment\": True,\n    \"cutout\": False,\n    \"max_epochs\": 20 # from time to time I'm correcting it\n}\n\n\ntrain_loader, val_loader, test_loader = get_cifar10_loaders_with_val(\n    batch_size=baseline_config[\"batch_size\"],\n    use_randaugment=baseline_config[\"randaugment\"],\n    cutout=baseline_config[\"cutout\"],\n    val_fraction=0.1\n)","metadata":{"execution":{"iopub.status.busy":"2025-11-14T17:01:53.736891Z","iopub.execute_input":"2025-11-14T17:01:53.737600Z","iopub.status.idle":"2025-11-14T17:01:55.872175Z","shell.execute_reply.started":"2025-11-14T17:01:53.737571Z","shell.execute_reply":"2025-11-14T17:01:55.871546Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = TinyViT(img_size=32,\n                patch_size=baseline_config[\"patch_size\"],\n                emb_dim=baseline_config[\"emb_dim\"],\n                mlp_dim=baseline_config[\"emb_dim\"]*baseline_config[\"mlp_mult\"],\n                heads=baseline_config[\"heads\"],\n                dropout=baseline_config[\"dropout\"]).to(device)\n\ncriterion = nn.CrossEntropyLoss(label_smoothing=baseline_config[\"label_smoothing\"]).to(device)\noptimizer = torch.optim.AdamW(model.parameters(), lr=baseline_config[\"lr\"], weight_decay=baseline_config[\"weight_decay\"])\n\nhistory = {\"epoch\": [], \"train_loss\": [], \"train_acc\": [], \"val_loss\": [], \"val_acc\": [], \"epoch_time\": []}\ntotal_train_time = 0.0\nbest_val_acc = 0.0\nbest_epoch = 0\n\nfor epoch in range(baseline_config[\"max_epochs\"]):\n    t_loss, t_acc, epoch_time = train_epoch(model, train_loader, criterion, optimizer, device,\n                                            use_mixup=(baseline_config[\"mixup_alpha\"]>0.0),\n                                            mixup_alpha=baseline_config[\"mixup_alpha\"])\n    v_loss, v_acc, _ = validate(model, val_loader, criterion, device)\n    total_train_time += epoch_time\n    history[\"epoch\"].append(epoch+1)\n    history[\"train_loss\"].append(t_loss)\n    history[\"train_acc\"].append(t_acc)\n    history[\"val_loss\"].append(v_loss)\n    history[\"val_acc\"].append(v_acc)\n    history[\"epoch_time\"].append(epoch_time)\n    if v_acc > best_val_acc:\n        best_val_acc = v_acc\n        best_epoch = epoch+1\n        # Here I'm saving the best validation checkpoint\n        torch.save({\n            \"model_state\": model.state_dict(),\n            \"optim_state\": optimizer.state_dict(),\n            \"epoch\": best_epoch\n        }, \"baseline_best_checkpoint.pt\")\n    print(f\"Epoch {epoch+1}/{baseline_config['max_epochs']} | train_acc={t_acc:.4f} | val_acc={v_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T17:01:56.378265Z","iopub.execute_input":"2025-11-14T17:01:56.378553Z","iopub.status.idle":"2025-11-14T17:09:53.727361Z","shell.execute_reply.started":"2025-11-14T17:01:56.378531Z","shell.execute_reply":"2025-11-14T17:09:53.726484Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Saving my model and history for future usage","metadata":{}},{"cell_type":"code","source":"os.makedirs(\"baseline_results\", exist_ok=True)\ntorch.save(model.state_dict(), \"baseline_results/tinyvit_baseline_final.pth\")\nwith open(\"baseline_results/history.json\", \"w\") as f: json.dump(history, f, indent=2)\n\nbaseline_summary = {\n    \"best_val_acc\": float(best_val_acc),\n    \"best_epoch\": int(best_epoch),\n    \"total_train_time_s\": float(total_train_time),\n    \"config\": baseline_config\n}\nwith open(\"baseline_results/summary.json\", \"w\") as f: json.dump(baseline_summary, f, indent=2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T17:09:53.729211Z","iopub.execute_input":"2025-11-14T17:09:53.729438Z","iopub.status.idle":"2025-11-14T17:09:53.738393Z","shell.execute_reply.started":"2025-11-14T17:09:53.729418Z","shell.execute_reply":"2025-11-14T17:09:53.737821Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's plot what we've got for baseline model","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8,4))\nplt.plot(history[\"epoch\"], history[\"train_acc\"], label=\"train_acc\")\nplt.plot(history[\"epoch\"], history[\"val_acc\"], label=\"val_acc\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy\")\nplt.title(\"Baseline TinyViT (1 layer) Accuracy - 100 Epochs\")\nplt.legend(); plt.grid(True); plt.tight_layout()\nplt.savefig(\"baseline_results/baseline_acc_plot.png\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T17:09:53.739236Z","iopub.execute_input":"2025-11-14T17:09:53.739489Z","iopub.status.idle":"2025-11-14T17:09:54.110674Z","shell.execute_reply.started":"2025-11-14T17:09:53.739464Z","shell.execute_reply":"2025-11-14T17:09:54.109865Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Here is the beginning of my Hyperparameter Optimization","metadata":{}},{"cell_type":"raw","source":"Here is my set of search space for HPO\nconfig[\"patch_size\"]   = random.choice([2, 4]) \\\nconfig[\"emb_dim\"]      = random.choice([64, 96, 128])   D size \\\nconfig[\"mlp_mult\"]     = random.choice([2, 3, 4])       MLP size = emb_dim * mlp_mult \\\nconfig[\"heads\"]        = random.choice([2, 4, 8]) \\\nconfig[\"dropout\"]      = random.choice([0.0, 0.1, 0.2, 0.3]) \\","metadata":{}},{"cell_type":"code","source":"def sample_random_config():\n    \"\"\"Sample a random configuration from predefined ranges.\"\"\"\n    config = {}\n    config[\"patch_size\"]   = random.choice([2, 4])\n    config[\"emb_dim\"]      = random.choice([64, 96, 128])\n    config[\"mlp_mult\"]     = random.choice([2, 3, 4])\n    config[\"heads\"]        = random.choice([2, 4, 8])\n    config[\"dropout\"]      = random.choice([0.0, 0.1, 0.2, 0.3])\n\n    lr_log_min, lr_log_max = math.log10(1e-4), math.log10(3e-3)\n    wd_log_min, wd_log_max = math.log10(1e-4), math.log10(1e-1)\n\n    config[\"lr\"] = 10 ** random.uniform(lr_log_min, lr_log_max)\n    config[\"weight_decay\"] = 10 ** random.uniform(wd_log_min, wd_log_max)\n\n    config[\"batch_size\"]   = random.choice([64, 128, 256])\n    config[\"mixup_alpha\"]  = random.choice([0.0, 0.1, 0.2, 0.3])\n    config[\"label_smoothing\"] = random.choice([0.0, 0.05, 0.1])\n    config[\"randaugment\"]  = random.choice([True, False])\n    config[\"cutout\"]       = random.choice([True, False])\n    config[\"max_epochs\"]   = 30  # from time to time I changing it, to see whether I can afford myself that time\n\n    return config","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T17:26:43.732111Z","iopub.execute_input":"2025-11-14T17:26:43.732842Z","iopub.status.idle":"2025-11-14T17:26:43.740137Z","shell.execute_reply.started":"2025-11-14T17:26:43.732811Z","shell.execute_reply":"2025-11-14T17:26:43.739223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_loaders_for_config(config):\n    train_loader, val_loader, test_loader = get_cifar10_loaders_with_val(\n        batch_size    = config[\"batch_size\"],\n        use_randaugment = config[\"randaugment\"],\n        cutout        = config[\"cutout\"],\n        val_fraction  = 0.1,\n        data_dir      = \"./data\",\n        num_workers   = 4\n    )\n    return train_loader, val_loader, test_loader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T17:26:45.026422Z","iopub.execute_input":"2025-11-14T17:26:45.027035Z","iopub.status.idle":"2025-11-14T17:26:45.031110Z","shell.execute_reply.started":"2025-11-14T17:26:45.027008Z","shell.execute_reply":"2025-11-14T17:26:45.030228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_model_for_config(config):\n    mlp_dim = config[\"emb_dim\"] * config[\"mlp_mult\"]\n    model = TinyViT(\n        img_size   = 32,\n        patch_size = config[\"patch_size\"],\n        in_ch      = 3,\n        emb_dim    = config[\"emb_dim\"],\n        mlp_dim    = mlp_dim,\n        heads      = config[\"heads\"],\n        dropout    = config[\"dropout\"],\n        num_classes = 10\n    ).to(device)\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T17:26:49.114841Z","iopub.execute_input":"2025-11-14T17:26:49.115144Z","iopub.status.idle":"2025-11-14T17:26:49.119711Z","shell.execute_reply.started":"2025-11-14T17:26:49.115122Z","shell.execute_reply":"2025-11-14T17:26:49.118753Z"}},"outputs":[],"execution_count":null},{"cell_type":"raw","source":"f_tuple = (\n        config[\"patch_size\"],  # Patch size\n        1,                     # Layers = 1 (fixed)\n        config[\"emb_dim\"],     # D size\n        config[\"emb_dim\"] * config[\"mlp_mult\"],  # MLP size\n        config[\"heads\"]        # Heads\n    )","metadata":{}},{"cell_type":"code","source":"def run_one_trial(trial_id, config):\n    print(f\"\\n========== Trial {trial_id} ==========\")\n    print(\"Config:\")\n    print(json.dumps(config, indent=2))\n\n    set_seed(42 + trial_id)\n\n    train_loader, val_loader, test_loader = build_loaders_for_config(config)\n\n    model = build_model_for_config(config)\n\n    criterion = nn.CrossEntropyLoss(\n        label_smoothing=config[\"label_smoothing\"]\n    ).to(device)\n\n    optimizer = torch.optim.AdamW(\n        model.parameters(),\n        lr = config[\"lr\"],\n        weight_decay = config[\"weight_decay\"]\n    )\n\n    history = {\n        \"epoch\": [], \"train_loss\": [], \"train_acc\": [],\n        \"val_loss\": [], \"val_acc\": [], \"epoch_time\": []\n    }\n\n    best_val_acc = 0.0\n    best_epoch   = 0\n\n    train_start_time = time.time()\n    for epoch in range(config[\"max_epochs\"]):\n        t_loss, t_acc, epoch_time = train_epoch(\n            model, train_loader, criterion, optimizer, device,\n            use_mixup  = (config[\"mixup_alpha\"] > 0.0),\n            mixup_alpha = config[\"mixup_alpha\"]\n        )\n        v_loss, v_acc, _ = validate(model, val_loader, criterion, device)\n\n        history[\"epoch\"].append(epoch + 1)\n        history[\"train_loss\"].append(t_loss)\n        history[\"train_acc\"].append(t_acc)\n        history[\"val_loss\"].append(v_loss)\n        history[\"val_acc\"].append(v_acc)\n        history[\"epoch_time\"].append(epoch_time)\n\n        if v_acc > best_val_acc:\n            best_val_acc = v_acc\n            best_epoch   = epoch + 1\n\n        print(f\"  Epoch {epoch+1}/{config['max_epochs']}: \"\n              f\"train_acc={t_acc:.4f}, val_acc={v_acc:.4f}\")\n\n    total_train_time = time.time() - train_start_time\n\n    val_loss, val_acc, _   = validate(model, val_loader, criterion, device)\n    test_loss, test_acc, _ = validate(model, test_loader, criterion, device)\n\n    model.eval()\n    inf_start = time.time()\n    with torch.no_grad():\n        for xb, yb in test_loader:\n            xb = xb.to(device)\n            _ = model(xb)\n    inference_time = time.time() - inf_start\n\n    result = {\n        \"trial_id\": trial_id,\n        \"config\": config,\n        \"best_val_acc\": float(best_val_acc),\n        \"best_epoch\": int(best_epoch),\n        \"final_val_acc\": float(val_acc),\n        \"test_acc\": float(test_acc),\n        \"total_train_time_s\": float(total_train_time),\n        \"inference_time_s\": float(inference_time),\n        \"history\": history\n    }\n\n    f_tuple = (\n        config[\"patch_size\"],\n        1,\n        config[\"emb_dim\"],\n        config[\"emb_dim\"] * config[\"mlp_mult\"],\n        config[\"heads\"]\n    )\n\n    print(\"\\n  Trial summary:\")\n    print(f\"    f = (Patch={f_tuple[0]}, Layers={f_tuple[1]}, \"\n          f\"D={f_tuple[2]}, MLP={f_tuple[3]}, Heads={f_tuple[4]})\")\n    print(f\"    Training HPs: lr={config['lr']:.5f}, \"\n          f\"weight_decay={config['weight_decay']:.5f}, \"\n          f\"batch_size={config['batch_size']}, \"\n          f\"dropout={config['dropout']}, \"\n          f\"mixup_alpha={config['mixup_alpha']}, \"\n          f\"label_smoothing={config['label_smoothing']}, \"\n          f\"randaugment={config['randaugment']}, cutout={config['cutout']}\")\n    print(f\"    Total training time: {total_train_time:.2f} s\")\n    print(f\"    Inference time (full test set): {inference_time:.2f} s\")\n    print(f\"    Best val acc: {best_val_acc:.4f} at epoch {best_epoch}\")\n    print(f\"    Final val acc: {val_acc:.4f}, Test acc: {test_acc:.4f}\")\n\n    return result","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T17:26:52.989842Z","iopub.execute_input":"2025-11-14T17:26:52.990553Z","iopub.status.idle":"2025-11-14T17:26:53.001026Z","shell.execute_reply.started":"2025-11-14T17:26:52.990532Z","shell.execute_reply":"2025-11-14T17:26:53.000216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_trials = 8 # I should change this one to 32 combinations\nresults = []\nbest_result = None\nbest_val_acc_overall = -1.0\n\nos.makedirs(\"random_search_results\", exist_ok=True)\n\nfor trial_id in range(1, num_trials + 1):\n    cfg = sample_random_config()\n    res = run_one_trial(trial_id, cfg)\n    results.append(res)\n\n    if res[\"best_val_acc\"] > best_val_acc_overall:\n        best_val_acc_overall = res[\"best_val_acc\"]\n        best_result = res\n\n        torch.save({\n            \"model_state\": build_model_for_config(cfg).state_dict()\n        }, \"random_search_results/best_tinyvit_randomsearch.pth\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T17:26:55.732155Z","iopub.execute_input":"2025-11-14T17:26:55.732440Z","iopub.status.idle":"2025-11-14T19:02:34.435753Z","shell.execute_reply.started":"2025-11-14T17:26:55.732420Z","shell.execute_reply":"2025-11-14T19:02:34.434995Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Visualization","metadata":{}},{"cell_type":"code","source":"records = []\nfor r in results:\n    cfg = r[\"config\"]\n    records.append({\n        \"trial_id\": r[\"trial_id\"],\n        \"best_val_acc\": r[\"best_val_acc\"],\n        \"test_acc\": r[\"test_acc\"],\n        \"train_time_s\": r[\"total_train_time_s\"],\n        \"inference_time_s\": r[\"inference_time_s\"],\n        \"patch_size\": cfg[\"patch_size\"],\n        \"emb_dim\": cfg[\"emb_dim\"],\n        \"mlp_mult\": cfg[\"mlp_mult\"],\n        \"heads\": cfg[\"heads\"],\n        \"dropout\": cfg[\"dropout\"],\n        \"lr\": cfg[\"lr\"],\n        \"weight_decay\": cfg[\"weight_decay\"],\n        \"batch_size\": cfg[\"batch_size\"],\n        \"mixup_alpha\": cfg[\"mixup_alpha\"],\n        \"label_smoothing\": cfg[\"label_smoothing\"],\n        \"randaugment\": cfg[\"randaugment\"],\n        \"cutout\": cfg[\"cutout\"],\n    })\n\ndf = pd.DataFrame.from_records(records)\ndf.to_csv(\"random_search_results/random_search_summary.csv\", index=False)\n\nprint(\"\\nRandom search summary (per trial):\")\nprint(df[[\n    \"trial_id\", \"best_val_acc\", \"test_acc\", \"train_time_s\",\n    \"patch_size\", \"emb_dim\", \"mlp_mult\", \"heads\", \"lr\", \"batch_size\"\n]])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T19:03:01.298840Z","iopub.execute_input":"2025-11-14T19:03:01.299636Z","iopub.status.idle":"2025-11-14T19:03:01.335772Z","shell.execute_reply.started":"2025-11-14T19:03:01.299605Z","shell.execute_reply":"2025-11-14T19:03:01.335184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8, 4))\nplt.bar(df[\"trial_id\"].astype(str), df[\"best_val_acc\"])\nplt.xlabel(\"Trial ID\")\nplt.ylabel(\"Best Validation Accuracy\")\nplt.title(\"Random Search: Best Val Accuracy per Trial\")\nplt.grid(True, axis=\"y\", linestyle=\"--\", alpha=0.5)\nplt.tight_layout()\nplt.savefig(\"random_search_results/best_val_acc_per_trial.png\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T19:03:04.323670Z","iopub.execute_input":"2025-11-14T19:03:04.323999Z","iopub.status.idle":"2025-11-14T19:03:04.580668Z","shell.execute_reply.started":"2025-11-14T19:03:04.323970Z","shell.execute_reply":"2025-11-14T19:03:04.580069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if best_result is not None:\n    hist = best_result[\"history\"]\n    plt.figure(figsize=(7, 4))\n    plt.plot(hist[\"epoch\"], hist[\"train_acc\"], marker=\"o\", label=\"Train acc\")\n    plt.plot(hist[\"epoch\"], hist[\"val_acc\"], marker=\"s\", label=\"Val acc\")\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"Accuracy\")\n    plt.title(f\"Best Trial #{best_result['trial_id']} Learning Curves\")\n    plt.legend()\n    plt.grid(True, linestyle=\"--\", alpha=0.5)\n    plt.tight_layout()\n    plt.savefig(\"random_search_results/best_trial_learning_curves.png\")\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T19:03:09.743115Z","iopub.execute_input":"2025-11-14T19:03:09.743809Z","iopub.status.idle":"2025-11-14T19:03:10.064326Z","shell.execute_reply.started":"2025-11-14T19:03:09.743784Z","shell.execute_reply":"2025-11-14T19:03:10.063446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"random_search_results/all_results.json\", \"w\") as f:\n    json.dump(results, f, indent=2)\n\nprint(\"\\n===================================\")\nprint(\"Short summary of first 5 trials:\")\nprint(\"===================================\")\nfor res in results[:5]:\n    cfg = res[\"config\"]\n    f_tuple = (\n        cfg[\"patch_size\"],\n        1,\n        cfg[\"emb_dim\"],\n        cfg[\"emb_dim\"] * cfg[\"mlp_mult\"],\n        cfg[\"heads\"]\n    )\n    print(f\"\\nTrial {res['trial_id']}:\")\n    print(f\"  f = (Patch={f_tuple[0]}, Layers={f_tuple[1]}, \"\n          f\"D={f_tuple[2]}, MLP={f_tuple[3]}, Heads={f_tuple[4]})\")\n    print(f\"  Training HPs: lr={cfg['lr']:.5f}, \"\n          f\"weight_decay={cfg['weight_decay']:.5f}, \"\n          f\"batch_size={cfg['batch_size']}, \"\n          f\"dropout={cfg['dropout']}, \"\n          f\"mixup_alpha={cfg['mixup_alpha']}, \"\n          f\"label_smoothing={cfg['label_smoothing']}, \"\n          f\"randaugment={cfg['randaugment']}, cutout={cfg['cutout']}\")\n    print(f\"  Best val acc: {res['best_val_acc']:.4f} (epoch {res['best_epoch']})\")\n    print(f\"  Final val acc: {res['final_val_acc']:.4f}, \"\n          f\"Test acc: {res['test_acc']:.4f}\")\n    print(f\"  Total training time: {res['total_train_time_s']:.2f} s\")\n    print(f\"  Inference time (test set): {res['inference_time_s']:.2f} s\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T19:03:18.221522Z","iopub.execute_input":"2025-11-14T19:03:18.222137Z","iopub.status.idle":"2025-11-14T19:03:18.232149Z","shell.execute_reply.started":"2025-11-14T19:03:18.222113Z","shell.execute_reply":"2025-11-14T19:03:18.231465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n===================================\")\nprint(\"Best configuration over all trials\")\nprint(\"===================================\")\nif best_result is not None:\n    cfg = best_result[\"config\"]\n    f_tuple = (\n        cfg[\"patch_size\"],\n        1,\n        cfg[\"emb_dim\"],\n        cfg[\"emb_dim\"] * cfg[\"mlp_mult\"],\n        cfg[\"heads\"]\n    )\n    print(f\"Best validation accuracy: {best_result['best_val_acc']:.4f} \"\n          f\"(epoch {best_result['best_epoch']})\")\n    print(f\"Corresponding test accuracy: {best_result['test_acc']:.4f}\")\n    print(f\"f = (Patch={f_tuple[0]}, Layers={f_tuple[1]}, \"\n          f\"D={f_tuple[2]}, MLP={f_tuple[3]}, Heads={f_tuple[4]})\")\n    print(\"Training HPs:\")\n    print(f\"  lr={cfg['lr']:.5f}, weight_decay={cfg['weight_decay']:.5f}\")\n    print(f\"  batch_size={cfg['batch_size']}, dropout={cfg['dropout']}\")\n    print(f\"  mixup_alpha={cfg['mixup_alpha']}, \"\n          f\"label_smoothing={cfg['label_smoothing']}\")\n    print(f\"  randaugment={cfg['randaugment']}, cutout={cfg['cutout']}\")\n    print(f\"  Total training time: {best_result['total_train_time_s']:.2f} s\")\n    print(f\"  Inference time (test set): {best_result['inference_time_s']:.2f} s\")\nelse:\n    print(\"No trials were run.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T19:03:23.740826Z","iopub.execute_input":"2025-11-14T19:03:23.741374Z","iopub.status.idle":"2025-11-14T19:03:23.748012Z","shell.execute_reply.started":"2025-11-14T19:03:23.741352Z","shell.execute_reply":"2025-11-14T19:03:23.747213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"records = []\nfor r in results:\n    cfg = r[\"config\"]\n    records.append({\n        \"trial_id\": r[\"trial_id\"],\n        \"best_val_acc\": r[\"best_val_acc\"],\n        \"test_acc\": r[\"test_acc\"],\n        \"total_train_time_s\": r[\"total_train_time_s\"],\n        \"inference_time_s\": r[\"inference_time_s\"],\n        \"patch_size\": cfg[\"patch_size\"],\n        \"emb_dim\": cfg[\"emb_dim\"],\n        \"mlp_mult\": cfg[\"mlp_mult\"],\n        \"heads\": cfg[\"heads\"],\n        \"dropout\": cfg[\"dropout\"],\n        \"lr\": cfg[\"lr\"],\n        \"weight_decay\": cfg[\"weight_decay\"],\n        \"batch_size\": cfg[\"batch_size\"],\n        \"mixup_alpha\": cfg[\"mixup_alpha\"],\n        \"label_smoothing\": cfg[\"label_smoothing\"],\n        \"randaugment\": cfg[\"randaugment\"],\n        \"cutout\": cfg[\"cutout\"],\n    })\n\ndf = pd.DataFrame.from_records(records)\n\ndf_sorted = df.sort_values(\"best_val_acc\", ascending=False)\n\ndf_top5 = df_sorted.head(5).reset_index(drop=True)\n\nprint(\"\\n===================================\")\nprint(\"Top-5 search spaces (by validation accuracy)\")\nprint(\"===================================\")\n\nfor idx, row in df_top5.iterrows():\n    f_tuple = (\n        row[\"patch_size\"],\n        1,  # Layers = 1 (fixed in TinyViT)\n        row[\"emb_dim\"],\n        row[\"emb_dim\"] * row[\"mlp_mult\"],\n        row[\"heads\"]\n    )\n    print(f\"\\nRank {idx+1} (Trial {row['trial_id']}):\")\n    print(f\"  f = (Patch={f_tuple[0]}, Layers={f_tuple[1]}, \"\n          f\"D={f_tuple[2]}, MLP={f_tuple[3]}, Heads={f_tuple[4]})\")\n    print(f\"  Training HPs:\")\n    print(f\"    lr={row['lr']:.5f}, weight_decay={row['weight_decay']:.5f}\")\n    print(f\"    batch_size={row['batch_size']}, dropout={row['dropout']}\")\n    print(f\"    mixup_alpha={row['mixup_alpha']}, \"\n          f\"label_smoothing={row['label_smoothing']}\")\n    print(f\"    randaugment={row['randaugment']}, cutout={row['cutout']}\")\n    print(f\"  Performance:\")\n    print(f\"    Best val acc = {row['best_val_acc']:.4f}\")\n    print(f\"    Test acc     = {row['test_acc']:.4f}\")\n    print(f\"    Total training time = {row['total_train_time_s']:.2f} s\")\n    print(f\"    Inference time (test set) = {row['inference_time_s']:.2f} s\")\n\nbest_among_top5 = df_top5.sort_values(\"test_acc\", ascending=False).iloc[0]\n\nprint(\"\\n===================================\")\nprint(\"Best TEST score among Top-5 search spaces\")\nprint(\"===================================\")\n\nf_best = (\n    best_among_top5[\"patch_size\"],\n    1,\n    best_among_top5[\"emb_dim\"],\n    best_among_top5[\"emb_dim\"] * best_among_top5[\"mlp_mult\"],\n    best_among_top5[\"heads\"]\n)\n\nprint(f\"Best test accuracy (among Top-5) = {best_among_top5['test_acc']:.4f}\")\nprint(f\"Achieved by Trial {int(best_among_top5['trial_id'])}\")\nprint(f\"f = (Patch={f_best[0]}, Layers={f_best[1]}, \"\n      f\"D={f_best[2]}, MLP={f_best[3]}, Heads={f_best[4]})\")\nprint(\"Training HPs:\")\nprint(f\"  lr={best_among_top5['lr']:.5f}, \"\n      f\"weight_decay={best_among_top5['weight_decay']:.5f}\")\nprint(f\"  batch_size={best_among_top5['batch_size']}, \"\n      f\"dropout={best_among_top5['dropout']}\")\nprint(f\"  mixup_alpha={best_among_top5['mixup_alpha']}, \"\n      f\"label_smoothing={best_among_top5['label_smoothing']}\")\nprint(f\"  randaugment={best_among_top5['randaugment']}, \"\n      f\"cutout={best_among_top5['cutout']}\")\nprint(f\"  Total training time: {best_among_top5['total_train_time_s']:.2f} s\")\nprint(f\"  Inference time (test set): {best_among_top5['inference_time_s']:.2f} s\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-14T19:03:27.102090Z","iopub.execute_input":"2025-11-14T19:03:27.102811Z","iopub.status.idle":"2025-11-14T19:03:27.121324Z","shell.execute_reply.started":"2025-11-14T19:03:27.102785Z","shell.execute_reply":"2025-11-14T19:03:27.120545Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## So the best accuracy is 55.48%","metadata":{}}]}