{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q timm py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:54:37.425098Z","iopub.execute_input":"2025-09-26T17:54:37.425782Z","iopub.status.idle":"2025-09-26T17:55:53.930773Z","shell.execute_reply.started":"2025-09-26T17:54:37.425755Z","shell.execute_reply":"2025-09-26T17:55:53.929978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n\n# file_path = \"/kaggle/working/best_model.pth\"\n\n# if os.path.exists(file_path):\n#     os.remove(file_path)\n#     print(\"Đã xóa:\", file_path)\n# else:\n#     print(\"Không tìm thấy file:\", file_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:06:35.653741Z","iopub.execute_input":"2025-09-13T06:06:35.654021Z","iopub.status.idle":"2025-09-13T06:06:35.658896Z","shell.execute_reply.started":"2025-09-13T06:06:35.653995Z","shell.execute_reply":"2025-09-13T06:06:35.658057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn as nn\nimport torch\nfrom torch.nn import functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\nimport torchvision\nimport torchvision.transforms as transforms\nfrom einops import rearrange, repeat\nimport os\nfrom einops.layers.torch import Rearrange\nfrom einops.layers.torch import Reduce\nimport numpy as np \nimport pandas as pd \nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport torchvision.transforms as T\nfrom tqdm.notebook import tqdm\nfrom PIL import Image \nimport py7zr\nimport matplotlib.pyplot as plt\nimport copy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:57:18.456449Z","iopub.execute_input":"2025-09-26T17:57:18.456848Z","iopub.status.idle":"2025-09-26T17:57:18.461966Z","shell.execute_reply.started":"2025-09-26T17:57:18.456827Z","shell.execute_reply":"2025-09-26T17:57:18.461126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MoELayer(nn.Module):\n    def __init__(self, experts, gate, k=1):\n        super().__init__()\n        assert len(experts) > 0\n        self.experts = nn.ModuleList(experts)\n        self.gate = gate\n        self.k = k\n        self.num_experts = len(experts)\n\n    def forward(self, input: torch.Tensor):\n        B, T, d = input.shape\n        inputs = input.view(B*T, d)\n        N, d = inputs.shape\n        gate_logits = self.gate(inputs)\n        topk_vals, topk_idx = torch.topk(gate_logits, self.k, dim=-1)  \n        topk_weights = F.softmax(topk_vals, dim=-1).type_as(inputs) \n    \n        results = torch.zeros_like(inputs) #Khởi tạo results \n        for expert_id, expert in enumerate(self.experts):\n            mask = (topk_idx == expert_id)\n            if not mask.any():\n                continue\n    \n            input_idx, slot_idx = mask.nonzero(as_tuple=True)\n            expert_in = inputs[input_idx]\n            expert_out = expert(expert_in)\n            w = topk_weights[input_idx, slot_idx]\n            results.index_add_(\n                0,\n                input_idx,\n                expert_out * w.unsqueeze(-1)\n            )\n    \n        probs = F.softmax(gate_logits, dim=-1)   # [N, num_experts]\n        expert_importance = probs.mean(0)        # [num_experts]\n        balancing_loss = -(expert_importance * torch.log(expert_importance + 1e-9)).sum()\n    \n        #reshape lại đúng [B, T, d]\n        return results.view(B, T, d), balancing_loss\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:57:21.641279Z","iopub.execute_input":"2025-09-26T17:57:21.641614Z","iopub.status.idle":"2025-09-26T17:57:21.649739Z","shell.execute_reply.started":"2025-09-26T17:57:21.641591Z","shell.execute_reply":"2025-09-26T17:57:21.648974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MultiHeadAttention(nn.Module):\n    def __init__(self, embed_dim, num_heads=8):\n        super().__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        if embed_dim % num_heads != 0:\n            raise ValueError(\n                f\"embed_dim {embed_dim} not divisible by num_heads {num_heads}\"\n            )\n        self.projection_dim = embed_dim // num_heads\n        self.query = nn.Linear(embed_dim, embed_dim)\n        self.key = nn.Linear(embed_dim, embed_dim)\n        self.value = nn.Linear(embed_dim, embed_dim)\n\n        self.out_proj = nn.Linear(embed_dim, embed_dim)\n\n    def forward(self, x):\n        B, T, C = x.shape  \n        q = self.query(x)   \n        k = self.key(x)\n        v = self.value(x)\n        q = q.view(B, T, self.num_heads, self.projection_dim).transpose(1, 2)\n        k = k.view(B, T, self.num_heads, self.projection_dim).transpose(1, 2)\n        v = v.view(B, T, self.num_heads, self.projection_dim).transpose(1, 2)\n\n        # Attention scores: (B, num_heads, T, T)\n        attn_scores = (q @ k.transpose(-2, -1)) / (self.projection_dim ** 0.5)\n        attn_weights = F.softmax(attn_scores, dim=-1)\n\n        # Weighted sum: (B, num_heads, T, head_dim)\n        attn_output = attn_weights @ v\n\n        # Gộp lại các heads: (B, T, C)\n        attn_output = attn_output.transpose(1, 2).contiguous().view(B, T, C)\n\n        # Project lại output\n        out = self.out_proj(attn_output)\n        return out\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:59:19.525319Z","iopub.execute_input":"2025-09-26T17:59:19.525662Z","iopub.status.idle":"2025-09-26T17:59:19.532664Z","shell.execute_reply.started":"2025-09-26T17:59:19.525639Z","shell.execute_reply":"2025-09-26T17:59:19.532026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FeedForward(nn.Module):\n    def __init__(self, n_embed,  dropout=0.3):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(n_embed, 4* n_embed),\n            nn.ReLU(),\n            nn.Linear(4 * n_embed, n_embed),\n            nn.Dropout(dropout)\n        )\n    def forward(self, x):\n        return self.net(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:59:27.945507Z","iopub.execute_input":"2025-09-26T17:59:27.945834Z","iopub.status.idle":"2025-09-26T17:59:27.950179Z","shell.execute_reply.started":"2025-09-26T17:59:27.945813Z","shell.execute_reply":"2025-09-26T17:59:27.949556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Block(nn.Module):\n    def __init__(self, n_embed, n_head, num_experts = 4, dropout=0.3):\n        super().__init__()\n        self.sa_head = MultiHeadAttention(n_embed, n_head)\n        self.ffw = MoELayer(\n            experts=[FeedForward(n_embed) for _ in range(num_experts)],\n            gate=nn.Linear(n_embed, num_experts, bias=False),\n        )\n        self.ln1 = nn.LayerNorm(n_embed)\n        self.ln2 = nn.LayerNorm(n_embed)\n        self.dropout = nn.Dropout(dropout)\n    def forward(self, x):\n        x = x + self.dropout(self.sa_head(self.ln1(x)))\n        out, aux_loss = self.ffw(self.ln2(x))\n        x = x + self.dropout(out)\n        return x, aux_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:59:34.92011Z","iopub.execute_input":"2025-09-26T17:59:34.920391Z","iopub.status.idle":"2025-09-26T17:59:34.925863Z","shell.execute_reply.started":"2025-09-26T17:59:34.920327Z","shell.execute_reply":"2025-09-26T17:59:34.925212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class VisionTransformer(nn.Module):\n    def __init__(\n        self,\n        image_size,\n        patch_size,\n        num_layers,\n        num_classes,\n        d_model,\n        num_heads,\n        mlp_dim=None, \n        channels=3,\n        num_experts=4,\n        k=1,\n        dropout=0.3,\n    ):\n        super().__init__()\n        if image_size % patch_size != 0:\n            raise ValueError(\"image_size must be divisible by patch_size\")\n\n        self.image_size = image_size\n        self.patch_size = patch_size\n        self.d_model = d_model\n        self.num_layers = num_layers\n        self.channels = channels\n        self.patch_dim = channels * patch_size * patch_size\n        self.num_patches = (image_size // patch_size) ** 2\n        self.dropout = dropout\n        self.patch_proj = nn.Conv2d(in_channels=channels, out_channels=d_model,\n                                    kernel_size=patch_size, stride=patch_size)\n        self.class_token = nn.Parameter(torch.zeros(1, 1, d_model))\n        self.pos_emb = nn.Parameter(torch.zeros(1, self.num_patches + 1, d_model))\n        self.enc_layers = nn.ModuleList([\n            Block(n_embed=d_model, n_head=num_heads, num_experts=num_experts, dropout=dropout)\n            for _ in range(num_layers)\n        ])\n        self.norm = nn.LayerNorm(d_model)\n        self.mlp_head = nn.Sequential(\n            nn.Linear(d_model, mlp_dim if mlp_dim is not None else d_model),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(mlp_dim if mlp_dim is not None else d_model, num_classes)\n        )\n\n        # init\n        nn.init.trunc_normal_(self.pos_emb, std=0.02)\n        nn.init.trunc_normal_(self.class_token, std=0.02)\n\n    def extract_patches(self, images):\n        B = images.shape[0]\n        x = self.patch_proj(images)  # [B, d_model, H_patch, W_patch]\n        x = x.flatten(2).transpose(1, 2)  # -> [B, num_patches, d_model]\n        return x\n    def forward(self, x):\n\n        B = x.shape[0]\n        patches = self.extract_patches(x)  # [B, num_patches, d_model]\n\n        class_tokens = self.class_token.expand(B, -1, -1)  # [B, 1, d_model]\n        x = torch.cat([class_tokens, patches], dim=1)      # [B, num_patches+1, d_model]\n        x = x + self.pos_emb\n\n        aux_loss = 0.0\n        for layer in self.enc_layers:\n            x, lb = layer(x)   # layer returns (x, balancing_loss)\n            aux_loss = aux_loss + lb\n\n        x = self.norm(x)\n        cls = x[:, 0]  # take class token\n        logits = self.mlp_head(cls)\n        return logits, aux_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:59:38.87143Z","iopub.execute_input":"2025-09-26T17:59:38.871686Z","iopub.status.idle":"2025-09-26T17:59:38.880452Z","shell.execute_reply.started":"2025-09-26T17:59:38.87167Z","shell.execute_reply":"2025-09-26T17:59:38.879704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n         \n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:59:47.57754Z","iopub.execute_input":"2025-09-26T17:59:47.577818Z","iopub.status.idle":"2025-09-26T17:59:47.58321Z","shell.execute_reply.started":"2025-09-26T17:59:47.577796Z","shell.execute_reply":"2025-09-26T17:59:47.582547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    data_path = '/kaggle/input/cifar-10/'\n    train_archive = os.path.join(data_path, 'train.7z')\n    test_archive = os.path.join(data_path, 'test.7z')\n    train_labels_csv = os.path.join(data_path, 'trainLabels.csv')\n    \n    extract_path = '/kaggle/working/cifar10/'\n    train_img_path = os.path.join(extract_path, 'train')\n    test_img_path = os.path.join(extract_path, 'test')\n    model_save_path = '/kaggle/working/best_model.pth'\n    model_save_path_v2 = '/kaggle/working/best_model_v2.pth'\n    img_size = 32\n    batch_size = 16\n    epochs = 50\n    label_smoothing = 0.1\n    validation_split = 0.1 # Use 10% of training data for validation\n    \n    # Hardware\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    # CIFAR-10 specific\n    num_classes = 10\n    class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\n\nprint(f\"Using device: {CFG.device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:59:49.853103Z","iopub.execute_input":"2025-09-26T17:59:49.853652Z","iopub.status.idle":"2025-09-26T17:59:49.916379Z","shell.execute_reply.started":"2025-09-26T17:59:49.853628Z","shell.execute_reply":"2025-09-26T17:59:49.91564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# os.makedirs(CFG.extract_path, exist_ok=True)\n\n# print(\"Extracting train.7z...\")\n# with py7zr.SevenZipFile(CFG.train_archive, mode='r') as z:\n#     z.extractall(path=CFG.extract_path)\n# print(\"Train images extracted.\")\n\n# print(\"Extracting test.7z...\")\n# with py7zr.SevenZipFile(CFG.test_archive, mode='r') as z:\n#     z.extractall(path=CFG.extract_path)\n# print(\"Test images extracted.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:06:45.05565Z","iopub.execute_input":"2025-09-13T06:06:45.055868Z","iopub.status.idle":"2025-09-13T06:06:45.072Z","shell.execute_reply.started":"2025-09-13T06:06:45.055852Z","shell.execute_reply":"2025-09-13T06:06:45.071492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(CFG.train_labels_csv)\nclass_to_index = {\n    name: i for i, \n    name in enumerate(CFG.class_names)\n}\ntrain_df['label_idx'] = train_df['label'].map(class_to_index)\nid_to_label = dict(zip(train_df['id'], train_df['label_idx']))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:59:54.641177Z","iopub.execute_input":"2025-09-26T17:59:54.641545Z","iopub.status.idle":"2025-09-26T17:59:54.696422Z","shell.execute_reply.started":"2025-09-26T17:59:54.641516Z","shell.execute_reply":"2025-09-26T17:59:54.695691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Cifar10Dataset(Dataset):\n    def __init__(\n        self, \n        img_dir,\n        image_ids,\n        labels_map = None,\n        transform = None\n    ):\n        self.img_dir = img_dir\n        self.image_ids = image_ids \n        self.labels_map = labels_map\n        self.transform = transform\n    def __len__(self):\n        return len(self.image_ids)\n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        img_path = os.path.join(self.img_dir, f\"{image_id}.png\")\n        image = Image.open(img_path).convert(\"RGB\")\n        if self.transform:\n            image = self.transform(image)\n\n        if self.labels_map:\n            label = self.labels_map[image_id]\n            return image, torch.tensor(label, dtype=torch.long)\n        else:\n            return image, image_id\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:59:57.129076Z","iopub.execute_input":"2025-09-26T17:59:57.129874Z","iopub.status.idle":"2025-09-26T17:59:57.137359Z","shell.execute_reply.started":"2025-09-26T17:59:57.129842Z","shell.execute_reply":"2025-09-26T17:59:57.136608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_transforms = {\n    'train': T.Compose([\n        T.RandomCrop(CFG.img_size, padding=4),\n        T.RandomHorizontalFlip(),\n        T.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n        T.RandomRotation(15),\n        T.TrivialAugmentWide(),\n        T.RandomErasing(p=0.25),\n        T.ToTensor(),\n        T.Normalize(mean=[0.4914, 0.4822, 0.4465],\n                    std=[0.2023, 0.1994, 0.2010])\n    ]),\n    'val': T.Compose([\n        T.Resize((CFG.img_size, CFG.img_size)),\n        T.ToTensor(),\n        T.Normalize(mean=[0.4914, 0.4822, 0.4465],\n                    std=[0.2023, 0.1994, 0.2010])\n    ]),\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:00:00.808435Z","iopub.execute_input":"2025-09-26T18:00:00.808713Z","iopub.status.idle":"2025-09-26T18:00:00.814462Z","shell.execute_reply.started":"2025-09-26T18:00:00.808691Z","shell.execute_reply":"2025-09-26T18:00:00.813746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train_ids = train_df['id'].values\nfull_dataset = Cifar10Dataset(CFG.train_img_path, all_train_ids, id_to_label)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:00:04.032775Z","iopub.execute_input":"2025-09-26T18:00:04.033028Z","iopub.status.idle":"2025-09-26T18:00:04.037285Z","shell.execute_reply.started":"2025-09-26T18:00:04.033008Z","shell.execute_reply":"2025-09-26T18:00:04.036411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_size = int(len(full_dataset) * CFG.validation_split)\ntrain_size = len(full_dataset) - val_size\ntrain_dataset, val_dataset = random_split(full_dataset, [train_size, val_size])\n\ntrain_dataset.dataset.transform = data_transforms['train']\nval_dataset.dataset.transform = data_transforms['val']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:00:07.032692Z","iopub.execute_input":"2025-09-26T18:00:07.033377Z","iopub.status.idle":"2025-09-26T18:00:07.041656Z","shell.execute_reply.started":"2025-09-26T18:00:07.033335Z","shell.execute_reply":"2025-09-26T18:00:07.041058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=os.cpu_count(), pin_memory=True)\nval_loader = DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=os.cpu_count(), pin_memory=True)\n\nprint(f\"Training samples: {len(train_dataset)}, Validation samples: {len(val_dataset)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:00:09.587649Z","iopub.execute_input":"2025-09-26T18:00:09.588097Z","iopub.status.idle":"2025-09-26T18:00:09.592998Z","shell.execute_reply.started":"2025-09-26T18:00:09.588074Z","shell.execute_reply":"2025-09-26T18:00:09.592239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Lấy 1 batch từ train_loader\nimages, labels = next(iter(train_loader))\n\nprint(\"Images shape:\", images.shape)   # [batch_size, 3, H, W]\nprint(\"Labels shape:\", labels.shape)   # [batch_size]\nprint(\"First 10 labels:\", labels[:10])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:00:11.541189Z","iopub.execute_input":"2025-09-26T18:00:11.541844Z","iopub.status.idle":"2025-09-26T18:00:11.913396Z","shell.execute_reply.started":"2025-09-26T18:00:11.541818Z","shell.execute_reply":"2025-09-26T18:00:11.91275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(model, train_loader, val_loader, cfg, aux_loss_weight=0.01, patience=10):\n\n    model = model.to(cfg.device)\n    criterion = nn.CrossEntropyLoss(label_smoothing=cfg.label_smoothing)\n    optimizer = optim.AdamW(model.parameters(), lr=3e-4, weight_decay=0.05)\n\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n        optimizer, T_max=cfg.epochs\n    )\n\n    history = {\"train_loss\": [], \"train_acc\": [], \"val_loss\": [], \"val_acc\": []}\n    best_val_acc = 0.0\n    best_model_wts = copy.deepcopy(model.state_dict())\n    no_improve_epochs = 0  \n    start_epoch = 0  \n\n    if os.path.exists(cfg.model_save_path):\n        checkpoint = torch.load(cfg.model_save_path, map_location=cfg.device)\n        model.load_state_dict(checkpoint[\"model_state\"])\n        optimizer.load_state_dict(checkpoint[\"optimizer_state\"])\n        best_val_acc = checkpoint[\"best_acc\"]\n        history = checkpoint.get(\"history\", history)\n        start_epoch = checkpoint[\"epoch\"]\n        best_model_wts = copy.deepcopy(model.state_dict())\n        print(f\"Resumed from epoch {start_epoch}, best_val_acc={best_val_acc:.2f}%\")\n\n    for epoch in range(start_epoch, cfg.epochs):\n        model.train()\n        running_loss, running_correct, total = 0.0, 0, 0\n        pbar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Train]\")\n\n        for xb, yb in pbar:\n            xb, yb = xb.to(cfg.device), yb.to(cfg.device)\n\n            logits, aux_loss = model(xb)\n            cls_loss = criterion(logits, yb)\n            loss = cls_loss + aux_loss_weight * aux_loss\n\n            optimizer.zero_grad(set_to_none=True)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            optimizer.step()\n\n            running_loss += loss.item() * xb.size(0)\n            _, preds = logits.max(1)\n            running_correct += preds.eq(yb).sum().item()\n            total += yb.size(0)\n\n            pbar.set_postfix(loss=loss.item())\n\n        train_loss = running_loss / total\n        train_acc = 100.0 * running_correct / total\n\n\n        model.eval()\n        val_loss, val_correct, val_total = 0.0, 0, 0\n        with torch.no_grad():\n            for xb, yb in val_loader:\n                xb, yb = xb.to(cfg.device), yb.to(cfg.device)\n\n                logits, aux_loss = model(xb)\n                cls_loss = criterion(logits, yb)\n                loss = cls_loss + aux_loss_weight * aux_loss\n\n                val_loss += loss.item() * xb.size(0)\n                _, preds = logits.max(1)\n                val_correct += preds.eq(yb).sum().item()\n                val_total += yb.size(0)\n\n        val_loss /= val_total\n        val_acc = 100.0 * val_correct / val_total\n\n        history[\"train_loss\"].append(train_loss)\n        history[\"train_acc\"].append(train_acc)\n        history[\"val_loss\"].append(val_loss)\n        history[\"val_acc\"].append(val_acc)\n\n        print(f\"[Epoch {epoch+1}/{cfg.epochs}] \"\n              f\"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}% \"\n              f\"Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.2f}%\")\n\n        scheduler.step()\n\n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            best_model_wts = copy.deepcopy(model.state_dict())\n            torch.save({\n                \"epoch\": epoch+1,\n                \"model_state\": model.state_dict(),\n                \"optimizer_state\": optimizer.state_dict(),\n                \"best_acc\": best_val_acc,\n                \"history\": history\n            }, cfg.model_save_path)\n            print(f\"Saved best model with val_acc={best_val_acc:.2f}%\")\n            no_improve_epochs = 0\n        else:\n            no_improve_epochs += 1\n            if no_improve_epochs >= patience:\n                print(f\"Early stopping at epoch {epoch+1}\")\n                break\n\n    model.load_state_dict(best_model_wts)\n    return history\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-13T06:06:45.568215Z","iopub.execute_input":"2025-09-13T06:06:45.56852Z","iopub.status.idle":"2025-09-13T06:06:45.580714Z","shell.execute_reply.started":"2025-09-13T06:06:45.568481Z","shell.execute_reply":"2025-09-13T06:06:45.579959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    model = VisionTransformer(\n        image_size=CFG.img_size,\n        patch_size=8,\n        num_layers=4,\n        num_classes=CFG.num_classes,\n        d_model=128,\n        num_heads=8,\n        mlp_dim=256,\n        channels=3,\n        num_experts=4,\n        dropout=0.3\n    )\n\n    # history = train_model(model, train_loader, val_loader, CFG)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:00:50.28486Z","iopub.execute_input":"2025-09-26T18:00:50.285432Z","iopub.status.idle":"2025-09-26T18:00:50.330009Z","shell.execute_reply.started":"2025-09-26T18:00:50.28541Z","shell.execute_reply":"2025-09-26T18:00:50.32946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport matplotlib.pyplot as plt\n\n# Load checkpoint\ncheckpoint = torch.load(\"/kaggle/working/best_model.pth\", map_location=\"cpu\")\n\n# Lấy history\nhistory = checkpoint.get(\"history\", None)\nif history is None:\n    raise ValueError(\"Checkpoint không có history\")\n\n# Vẽ loss\nplt.figure(figsize=(10,5))\nplt.plot(history[\"train_loss\"], label=\"Train Loss\")\nplt.plot(history[\"val_loss\"], label=\"Validation Loss\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.title(\"Training & Validation Loss\")\nplt.legend()\nplt.show()\n\n# Nếu có accuracy thì vẽ thêm\nif \"train_acc\" in history and \"val_acc\" in history:\n    plt.figure(figsize=(10,5))\n    plt.plot(history[\"train_acc\"], label=\"Train Acc\")\n    plt.plot(history[\"val_acc\"], label=\"Validation Acc\")\n    plt.xlabel(\"Epoch\")\n    plt.ylabel(\"Accuracy\")\n    plt.title(\"Training & Validation Accuracy\")\n    plt.legend()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:00:26.186712Z","iopub.execute_input":"2025-09-26T18:00:26.187003Z","iopub.status.idle":"2025-09-26T18:00:26.612193Z","shell.execute_reply.started":"2025-09-26T18:00:26.186975Z","shell.execute_reply":"2025-09-26T18:00:26.611521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_acc = checkpoint.get(\"best_acc\", None)\nprint(best_acc)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:03:19.637403Z","iopub.execute_input":"2025-09-26T18:03:19.637949Z","iopub.status.idle":"2025-09-26T18:03:19.642015Z","shell.execute_reply.started":"2025-09-26T18:03:19.637924Z","shell.execute_reply":"2025-09-26T18:03:19.641374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nclasses = ['plane', 'car', 'bird', 'cat', 'deer',\n           'dog', 'frog', 'horse', 'ship', 'truck']\n\n\ndef imshow(img):\n    img = img / 2 + 0.5\n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n    plt.axis(\"off\")\n\n\ndef visualize_predictions(model, dataloader, device, num_images=50):\n    model = model.to(device)   # đảm bảo model ở đúng device\n    model.eval()\n    shown = 0\n    fig = plt.figure(figsize=(20, 10))\n\n    with torch.no_grad():\n        for images, labels in dataloader:\n            images, labels = images.to(device), labels.to(device)\n            outputs, _ = model(images)   \n            _, preds = torch.max(outputs, 1)\n\n            for j in range(images.size(0)):\n                if shown >= num_images:\n                    plt.show()\n                    return\n                shown += 1\n                ax = plt.subplot(5, 10, shown)\n                ax.axis(\"off\")\n                imshow(images[j].cpu())\n                true_label = classes[labels[j]]\n                pred_label = classes[preds[j]]\n                color = \"green\" if preds[j] == labels[j] else \"red\"\n                ax.set_title(f\"T:{true_label}\\nP:{pred_label}\", fontsize=8, color=color)\n\n\n\nvisualize_predictions(model, val_loader, CFG.device, num_images=50)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T18:04:03.543795Z","iopub.execute_input":"2025-09-26T18:04:03.544436Z","iopub.status.idle":"2025-09-26T18:04:05.769699Z","shell.execute_reply.started":"2025-09-26T18:04:03.544381Z","shell.execute_reply":"2025-09-26T18:04:05.768783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}