{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":37852,"databundleVersionId":4070249},{"sourceType":"modelInstanceVersion","sourceId":863794,"databundleVersionId":17147940,"modelInstanceId":656766,"modelId":668760}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt \nimport matplotlib.image as mpimg\nimport torch\n\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\n\nTRAIN_DIR = \"/kaggle/input/competitions/boolart-image-classification/train_image\"\nTEST_DIR = \"/kaggle/input/competitions/boolart-image-classification/test_image\"\nSAMPLE_SUBMISSION_DIR = \"/kaggle/input/competitions/boolart-image-classification/sample_submission.csv\"\nTRAIN_CSV_DIR = \"/kaggle/input/competitions/boolart-image-classification/train.csv\"\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using {DEVICE}.\")\n\n# 1. Read data & Visualize","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:30.373948Z","iopub.execute_input":"2026-05-13T16:11:30.374276Z","iopub.status.idle":"2026-05-13T16:11:43.468500Z","shell.execute_reply.started":"2026-05-13T16:11:30.374249Z","shell.execute_reply":"2026-05-13T16:11:43.467697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_paths = os.listdir(TRAIN_DIR)\ntest_paths = os.listdir(TEST_DIR)\n\nprint(f\"Total train images: {len(train_paths)}\")\nprint(f\"Total test images: {len(test_paths)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:43.469790Z","iopub.execute_input":"2026-05-13T16:11:43.470272Z","iopub.status.idle":"2026-05-13T16:11:43.908954Z","shell.execute_reply.started":"2026-05-13T16:11:43.470245Z","shell.execute_reply":"2026-05-13T16:11:43.908081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv = pd.read_csv(TRAIN_CSV_DIR)\nprint(f\"Number of rows: {len(train_csv)}\")\nprint(\"First 5 rows\")\nprint(train_csv.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:43.910084Z","iopub.execute_input":"2026-05-13T16:11:43.910406Z","iopub.status.idle":"2026-05-13T16:11:43.973933Z","shell.execute_reply.started":"2026-05-13T16:11:43.910364Z","shell.execute_reply":"2026-05-13T16:11:43.973208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Target's distribution\")\ntrain_csv[\"target\"].value_counts() * 100 / len(train_csv)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:43.975710Z","iopub.execute_input":"2026-05-13T16:11:43.976050Z","iopub.status.idle":"2026-05-13T16:11:43.998232Z","shell.execute_reply.started":"2026-05-13T16:11:43.976026Z","shell.execute_reply":"2026-05-13T16:11:43.997282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (10, 10))\nfor i in range(9):\n    plt.subplot(3, 3, i + 1)\n    path = os.path.join(TRAIN_DIR, train_paths[i])\n    img = mpimg.imread(path)\n    print(img.shape)\n    plt.imshow(img)\n    plt.axis(\"off\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:43.999229Z","iopub.execute_input":"2026-05-13T16:11:43.999537Z","iopub.status.idle":"2026-05-13T16:11:44.323982Z","shell.execute_reply.started":"2026-05-13T16:11:43.999501Z","shell.execute_reply":"2026-05-13T16:11:44.322962Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Build Transform pipeline & Dataset & DataLoader","metadata":{}},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(p = 0.5),\n    transforms.RandomVerticalFlip(p = 0.5),\n    transforms.RandomRotation(degrees = 15),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\nval_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.325173Z","iopub.execute_input":"2026-05-13T16:11:44.325963Z","iopub.status.idle":"2026-05-13T16:11:44.331397Z","shell.execute_reply.started":"2026-05-13T16:11:44.325934Z","shell.execute_reply":"2026-05-13T16:11:44.330543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, image_path : list[str] = train_paths, label_csv : pd.DataFrame = train_csv, transform = None):\n        self.image_path = image_path\n        self.label = dict(zip(label_csv[\"id\"], label_csv[\"target\"]))\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_path)\n\n    def __getitem__(self, index : int):\n        path = self.image_path[index]\n        img_id = int(path.split(\".\")[0])\n        label = self.label[img_id]\n\n        full_path = os.path.join(TRAIN_DIR, path)\n        try:\n            image = Image.open(full_path).convert(\"RGB\")\n        except Exception as e:\n            image = None\n            print(f\"Error at index {index}\")\n            print(f\"Error : {e}\")\n\n        if image:\n            if self.transform:\n                image = self.transform(image)\n            return image, label\n\nclass TestDataset(Dataset):\n    def __init__(self, image_path : list[str] = test_paths, transform = None):\n        self.image_path = image_path\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_path)\n\n    def __getitem__(self, index : int):\n        path = self.image_path[index]\n        full_path = os.path.join(TEST_DIR, path)\n        img_id = int(path.split(\".\")[0])\n\n        try:\n            image = Image.open(full_path).convert(\"RGB\")\n        except Exception as e:\n            image = None\n            print(f\"Error at index {index}\")\n            print(f\"Error: {e}\")\n\n        if image:\n            if self.transform:\n                image = self.transform(image)\n            return image, img_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.332322Z","iopub.execute_input":"2026-05-13T16:11:44.332829Z","iopub.status.idle":"2026-05-13T16:11:44.353678Z","shell.execute_reply.started":"2026-05-13T16:11:44.332790Z","shell.execute_reply":"2026-05-13T16:11:44.352997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ratio = 0.95\nindex_to_split = int(len(train_paths) * train_ratio)\ntrain_paths_split = train_paths[ : index_to_split]\nval_paths_split = train_paths[index_to_split : ]\nprint(f\"Train: {len(train_paths_split)} samples\")\nprint(f\"Val: {len(val_paths_split)} samples\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.354505Z","iopub.execute_input":"2026-05-13T16:11:44.354911Z","iopub.status.idle":"2026-05-13T16:11:44.375563Z","shell.execute_reply.started":"2026-05-13T16:11:44.354886Z","shell.execute_reply":"2026-05-13T16:11:44.374859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = TrainDataset(image_path = train_paths_split, transform = train_transform)\nval_dataset = TrainDataset(image_path = val_paths_split, transform = val_transform)\ntest_dataset = TestDataset(transform = val_transform)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.376518Z","iopub.execute_input":"2026-05-13T16:11:44.376800Z","iopub.status.idle":"2026-05-13T16:11:44.409111Z","shell.execute_reply.started":"2026-05-13T16:11:44.376764Z","shell.execute_reply":"2026-05-13T16:11:44.408463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BATCH_SIZE = 64\ntrain_dataloader = DataLoader(train_dataset, batch_size = BATCH_SIZE, shuffle = True)\nval_dataloader = DataLoader(val_dataset, batch_size = BATCH_SIZE)\ntest_dataloader = DataLoader(test_dataset, batch_size = BATCH_SIZE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.412254Z","iopub.execute_input":"2026-05-13T16:11:44.412544Z","iopub.status.idle":"2026-05-13T16:11:44.429662Z","shell.execute_reply.started":"2026-05-13T16:11:44.412485Z","shell.execute_reply":"2026-05-13T16:11:44.429075Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Train & Evaluate","metadata":{}},{"cell_type":"markdown","source":"## 3.1. Helpers","metadata":{}},{"cell_type":"code","source":"import time","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.431402Z","iopub.execute_input":"2026-05-13T16:11:44.431930Z","iopub.status.idle":"2026-05-13T16:11:44.446634Z","shell.execute_reply.started":"2026-05-13T16:11:44.431904Z","shell.execute_reply":"2026-05-13T16:11:44.445996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, dataloader, criterion, optimizer, lr_scheduler = None, device = DEVICE):\n    model.train()\n    train_loss = 0\n\n    for X, y in dataloader:\n        X = X.to(device)\n        y = y.to(device)\n        \n        optimizer.zero_grad()\n        logits = model(X)\n        loss = criterion(logits, y)\n\n        train_loss += loss.item()\n        loss.backward()\n        optimizer.step()\n\n    if lr_scheduler:\n        lr_scheduler.step()\n\n    return train_loss / len(dataloader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.447600Z","iopub.execute_input":"2026-05-13T16:11:44.447942Z","iopub.status.idle":"2026-05-13T16:11:44.462704Z","shell.execute_reply.started":"2026-05-13T16:11:44.447919Z","shell.execute_reply":"2026-05-13T16:11:44.462067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate(model, dataloader, criterion, device = DEVICE):\n    model.eval()\n    val_loss = 0\n\n    with torch.inference_mode():\n        for X, y in dataloader:\n            X = X.to(device)\n            y = y.to(device)\n\n            logits = model(X)\n            loss = criterion(logits, y)\n            val_loss += loss.item()\n\n    return val_loss / len(dataloader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.463469Z","iopub.execute_input":"2026-05-13T16:11:44.463725Z","iopub.status.idle":"2026-05-13T16:11:44.478691Z","shell.execute_reply.started":"2026-05-13T16:11:44.463692Z","shell.execute_reply":"2026-05-13T16:11:44.478038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def submit(model, dataloader, submission_name = None, device = DEVICE):\n    model.eval()\n\n    sample_submission = pd.read_csv(SAMPLE_SUBMISSION_DIR)\n    all_ids = []\n    all_preds = []\n    \n    with torch.inference_mode():\n        for batch_img, batch_img_id in dataloader:\n            batch_img = batch_img.to(device)\n            logits = model(batch_img)\n            labels = torch.argmax(logits, dim = 1)\n\n            all_ids.extend(batch_img_id.cpu().numpy())\n            all_preds.extend(labels.cpu().numpy())\n\n    pred_df = pd.DataFrame({\n        \"id\" : all_ids,\n        \"predict\" : all_preds\n    })\n\n    sample_submission = sample_submission.drop(columns = \"predict\")\n    sample_submission = pd.merge(sample_submission, pred_df, on = \"id\", how = \"left\")\n    \n    print(\"First 5 rows:\")\n    print(sample_submission.head())\n\n    if submission_name:\n        sample_submission.to_csv(submission_name + \"_submission.csv\", index = False)\n    else:\n        sample_submission.to_csv(\"submission.csv\", index = False)\n    print(\"Done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.479459Z","iopub.execute_input":"2026-05-13T16:11:44.480355Z","iopub.status.idle":"2026-05-13T16:11:44.496641Z","shell.execute_reply.started":"2026-05-13T16:11:44.480321Z","shell.execute_reply":"2026-05-13T16:11:44.496069Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2. Models","metadata":{}},{"cell_type":"code","source":"import torchvision.models as models\nimport torch.nn as nn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.497592Z","iopub.execute_input":"2026-05-13T16:11:44.497888Z","iopub.status.idle":"2026-05-13T16:11:44.517044Z","shell.execute_reply.started":"2026-05-13T16:11:44.497853Z","shell.execute_reply":"2026-05-13T16:11:44.516334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"out_features = 44","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.518162Z","iopub.execute_input":"2026-05-13T16:11:44.518417Z","iopub.status.idle":"2026-05-13T16:11:44.533986Z","shell.execute_reply.started":"2026-05-13T16:11:44.518394Z","shell.execute_reply":"2026-05-13T16:11:44.533402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_path = \"/kaggle/working/models\"\n\nos.makedirs(model_path, exist_ok = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.535107Z","iopub.execute_input":"2026-05-13T16:11:44.535403Z","iopub.status.idle":"2026-05-13T16:11:44.551428Z","shell.execute_reply.started":"2026-05-13T16:11:44.535381Z","shell.execute_reply":"2026-05-13T16:11:44.550518Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 1. Mobilenet v3 small","metadata":{}},{"cell_type":"code","source":"# mobilenet_v3_small = models.mobilenet_v3_small(weights = models.MobileNet_V3_Small_Weights.DEFAULT)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.552395Z","iopub.execute_input":"2026-05-13T16:11:44.552699Z","iopub.status.idle":"2026-05-13T16:11:44.568967Z","shell.execute_reply.started":"2026-05-13T16:11:44.552677Z","shell.execute_reply":"2026-05-13T16:11:44.567875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# last_layer = mobilenet_v3_small.classifier[-1]\n# in_features = last_layer.in_features\n# mobilenet_v3_small.classifier[-1] = nn.Linear(in_features, out_features)\n# print(mobilenet_v3_small.classifier)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.570310Z","iopub.execute_input":"2026-05-13T16:11:44.570904Z","iopub.status.idle":"2026-05-13T16:11:44.583865Z","shell.execute_reply.started":"2026-05-13T16:11:44.570877Z","shell.execute_reply":"2026-05-13T16:11:44.582910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for params in mobilenet_v3_small.parameters():\n#     params.requires_grad = False\n# for params in mobilenet_v3_small.classifier.parameters():\n#     params.requires_grad = True\n\n# mobilenet_v3_small_path = os.path.join(model_path, \"mobilenet_v3_small.pth\")\n# mobilenet_v3_small_device = mobilenet_v3_small.to(DEVICE)\n# criterion = nn.CrossEntropyLoss()\n# optimizer_stage_1 = torch.optim.Adam(mobilenet_v3_small_device.classifier.parameters(), lr = 1e-3)\n# print(\"Setup stage 1 completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.586166Z","iopub.execute_input":"2026-05-13T16:11:44.586726Z","iopub.status.idle":"2026-05-13T16:11:44.599738Z","shell.execute_reply.started":"2026-05-13T16:11:44.586687Z","shell.execute_reply":"2026-05-13T16:11:44.599082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EPOCHS_S1 = 5\n# train_loss_s1 = []\n# val_loss_s1 = []\nbest_val_loss = float(\"inf\")\n\n# print(\"======STAGE 1======\")\n# for e in range(EPOCHS_S1):\n#     print(f\"EPOCH [{e + 1}/{EPOCHS_S1}]: \", end = \"\")\n\n#     start = time.time()\n#     train_loss = train_one_epoch(mobilenet_v3_small_device, train_dataloader, criterion, optimizer_stage_1)\n#     val_loss = evaluate(mobilenet_v3_small_device, val_dataloader, criterion)\n#     end = time.time()\n\n#     dur = round(end - start)\n#     print(f\"train_loss = {train_loss} | val_loss = {val_loss} | time = {dur}s\")\n#     if val_loss < best_val_loss:\n#         best_val_loss = val_loss\n#         torch.save(mobilenet_v3_small_device.state_dict(), mobilenet_v3_small_path)\n#         print(f\">>Saved model's parameters to {mobilenet_v3_small_path}\")\n    \n#     train_loss_s1.append(train_loss)\n#     val_loss_s1.append(val_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.601288Z","iopub.execute_input":"2026-05-13T16:11:44.601783Z","iopub.status.idle":"2026-05-13T16:11:44.617501Z","shell.execute_reply.started":"2026-05-13T16:11:44.601711Z","shell.execute_reply":"2026-05-13T16:11:44.616437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for params in mobilenet_v3_small_device.parameters():\n#     params.requires_grad = True\n\n# mobilenet_v3_small_device.load_state_dict(torch.load(mobilenet_v3_small_path))\n# optimizer_stage_2 = torch.optim.Adam(mobilenet_v3_small_device.parameters(), lr = 1e-5)\n# print(\"Setup stage 2 completed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.618630Z","iopub.execute_input":"2026-05-13T16:11:44.619511Z","iopub.status.idle":"2026-05-13T16:11:44.641882Z","shell.execute_reply.started":"2026-05-13T16:11:44.619476Z","shell.execute_reply":"2026-05-13T16:11:44.640932Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EPOCHS_S2 = 5\n# train_loss_s2 = []\n# val_loss_s2 = []\n\n# print(\"======STAGE 2======\")\n# for e in range(EPOCHS_S2):\n#     print(f\"EPOCH [{e + 1}/{EPOCHS_S2}]: \", end = \"\")\n\n#     start = time.time()\n#     train_loss = train_one_epoch(mobilenet_v3_small_device, train_dataloader, criterion, optimizer_stage_2)\n#     val_loss = evaluate(mobilenet_v3_small_device, val_dataloader, criterion)\n#     end = time.time()\n\n#     dur = round(end - start)\n#     print(f\"train_loss = {train_loss} | val_loss = {val_loss} | time = {dur}s\")\n#     if val_loss < best_val_loss:\n#         best_val_loss = val_loss\n#         torch.save(mobilenet_v3_small_device.state_dict(), mobilenet_v3_small_path)\n#         print(f\">>Saved model's parameters to {mobilenet_v3_small_path}\")\n\n#     train_loss_s2.append(train_loss)\n#     val_loss_s2.append(val_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.642861Z","iopub.execute_input":"2026-05-13T16:11:44.643278Z","iopub.status.idle":"2026-05-13T16:11:44.660946Z","shell.execute_reply.started":"2026-05-13T16:11:44.643237Z","shell.execute_reply":"2026-05-13T16:11:44.659887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# mobilenet_v3_small_device.load_state_dict(torch.load(mobilenet_v3_small_path))\n\n# print(\"=====STAGE 3======\")\n# submit(mobilenet_v3_small_device, test_dataloader, submission_name = \"mobilenet_v3_small\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.662048Z","iopub.execute_input":"2026-05-13T16:11:44.662736Z","iopub.status.idle":"2026-05-13T16:11:44.675978Z","shell.execute_reply.started":"2026-05-13T16:11:44.662692Z","shell.execute_reply":"2026-05-13T16:11:44.675353Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. EfficientNet b2","metadata":{}},{"cell_type":"code","source":"# efficientnet_b2 = models.efficientnet_b2(weights = models.EfficientNet_B2_Weights.DEFAULT)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.676976Z","iopub.execute_input":"2026-05-13T16:11:44.677353Z","iopub.status.idle":"2026-05-13T16:11:44.696853Z","shell.execute_reply.started":"2026-05-13T16:11:44.677328Z","shell.execute_reply":"2026-05-13T16:11:44.696081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# last_layer = efficientnet_b2.classifier[-1]\n# in_features = last_layer.in_features\n# efficientnet_b2.classifier[-1] = nn.Linear(in_features, out_features)\n# print(efficientnet_b2.classifier)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.697797Z","iopub.execute_input":"2026-05-13T16:11:44.698328Z","iopub.status.idle":"2026-05-13T16:11:44.713067Z","shell.execute_reply.started":"2026-05-13T16:11:44.698290Z","shell.execute_reply":"2026-05-13T16:11:44.712174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for params in efficientnet_b2.parameters():\n#     params.requires_grad = False\n# for params in efficientnet_b2.classifier.parameters():\n#     params.requires_grad = True\n\n# efficientnet_b2_path = os.path.join(model_path, \"efficientnet_b2.pth\")\n# efficientnet_b2_device = efficientnet_b2.to(DEVICE)\n# criterion = nn.CrossEntropyLoss()\n# optimizer_stage_1 = torch.optim.Adam(efficientnet_b2_device.classifier.parameters(), lr = 1e-3)\n# print(\"Setup stage 1 completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.714070Z","iopub.execute_input":"2026-05-13T16:11:44.714338Z","iopub.status.idle":"2026-05-13T16:11:44.727474Z","shell.execute_reply.started":"2026-05-13T16:11:44.714315Z","shell.execute_reply":"2026-05-13T16:11:44.726860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EPOCHS_S1 = 5\n# train_loss_s1 = []\n# val_loss_s1 = []\n# best_val_loss = float(\"inf\")\n\n# print(\"======STAGE 1======\")\n# for e in range(EPOCHS_S1):\n#     print(f\"EPOCH [{e + 1}/{EPOCHS_S1}]: \", end = \"\")\n\n#    start = time.time()\n#    train_loss = train_one_epoch(efficientnet_b2_device, train_dataloader, criterion, optimizer_stage_1)\n#    val_loss = evaluate(efficientnet_b2_device, val_dataloader, criterion)\n#    end = time.time()\n\n#    dur = round(end - start)\n#    print(f\"train_loss = {train_loss} | val_loss = {val_loss} | time = {dur}s\")\n#    if val_loss < best_val_loss:\n#        best_val_loss = val_loss\n#        torch.save(efficientnet_b2_device.state_dict(), efficientnet_b2_path)\n#        print(f\">>Saved model's parameters to {efficientnet_b2_path}\")\n    \n#    train_loss_s1.append(train_loss)\n#    val_loss_s1.append(val_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.730445Z","iopub.execute_input":"2026-05-13T16:11:44.730759Z","iopub.status.idle":"2026-05-13T16:11:44.743210Z","shell.execute_reply.started":"2026-05-13T16:11:44.730735Z","shell.execute_reply":"2026-05-13T16:11:44.742447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# efficientnet_b2_device.load_state_dict(torch.load(\"/kaggle/input/models/daoviet/efficientnet-b2-boolart/pytorch/default/1/efficientnet_b2 (2).pth\"))\n# print(\"Load model successfully\")\n\n# for params in efficientnet_b2_device.parameters():\n#     params.requires_grad = True\n\n# EPOCHS_S2 = 15\n# train_loss_s2 = []\n# val_loss_s2 = []\n\n# optimizer_stage_2 = torch.optim.Adam(efficientnet_b2_device.parameters(), lr = 1e-5)\n# lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n#     optimizer_stage_2,\n#     T_max= EPOCHS_S2,\n#     eta_min = 1e-7\n# )\n# print(\"Setup stage 2 completed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.744146Z","iopub.execute_input":"2026-05-13T16:11:44.744441Z","iopub.status.idle":"2026-05-13T16:11:44.758818Z","shell.execute_reply.started":"2026-05-13T16:11:44.744407Z","shell.execute_reply":"2026-05-13T16:11:44.757906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(\"======STAGE 2======\")\n# for e in range(EPOCHS_S2):\n#     print(f\"EPOCH [{e + 1}/{EPOCHS_S2}]: \", end = \"\")\n\n#     start = time.time()\n#     train_loss = train_one_epoch(efficientnet_b2_device, train_dataloader, criterion, optimizer_stage_2, lr_scheduler)\n#     val_loss = evaluate(efficientnet_b2_device, val_dataloader, criterion)\n#     end = time.time()\n\n#     dur = round(end - start)\n#     print(f\"train_loss = {train_loss} | val_loss = {val_loss} | time = {dur}s\")\n#     if val_loss < best_val_loss:\n#         best_val_loss = val_loss\n#         torch.save(efficientnet_b2_device.state_dict(), efficientnet_b2_path)\n#         print(f\">>Saved model's parameters to {efficientnet_b2_path}\")\n\n#     train_loss_s2.append(train_loss)\n#     val_loss_s2.append(val_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.759932Z","iopub.execute_input":"2026-05-13T16:11:44.760366Z","iopub.status.idle":"2026-05-13T16:11:44.778724Z","shell.execute_reply.started":"2026-05-13T16:11:44.760341Z","shell.execute_reply":"2026-05-13T16:11:44.778138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# efficientnet_b2_device.load_state_dict(torch.load(efficientnet_b2_path))\n\n# print(\"=====STAGE 3======\")\n# submit(efficientnet_b2_device, test_dataloader, submission_name = \"efficientnet_b2\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.779733Z","iopub.execute_input":"2026-05-13T16:11:44.780133Z","iopub.status.idle":"2026-05-13T16:11:44.793616Z","shell.execute_reply.started":"2026-05-13T16:11:44.780098Z","shell.execute_reply":"2026-05-13T16:11:44.792961Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 3. ViT b16","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport os, time\n\n# ======LOAD MODEL======\nvit_b16 = models.vit_b_16(weights=models.ViT_B_16_Weights.DEFAULT)\nlast_layer = vit_b16.heads.head\nin_features = last_layer.in_features\nvit_b16.heads.head = nn.Linear(in_features, out_features)\nprint(vit_b16.heads)\n\n# ======STAGE 1: Train classifier only======\nfor params in vit_b16.parameters():\n    params.requires_grad = False\nfor params in vit_b16.heads.parameters():\n    params.requires_grad = True\n\nvit_b16_path = os.path.join(model_path, \"vit_b16.pth\")\nvit_b16_device = vit_b16.to(DEVICE)\ncriterion = nn.CrossEntropyLoss()\noptimizer_stage_1 = torch.optim.Adam(vit_b16_device.heads.parameters(), lr=1e-3)\nprint(\"Setup stage 1 completed\")\n\nEPOCHS_S1 = 5\ntrain_loss_s1, val_loss_s1 = [], []\nbest_val_loss = float(\"inf\")\n\nprint(\"======STAGE 1======\")\nfor e in range(EPOCHS_S1):\n    print(f\"EPOCH [{e + 1}/{EPOCHS_S1}]: \", end=\"\")\n\n    start = time.time()\n    train_loss = train_one_epoch(vit_b16_device, train_dataloader, criterion, optimizer_stage_1)\n    val_loss = evaluate(vit_b16_device, val_dataloader, criterion)\n    end = time.time()\n\n    dur = round(end - start)\n    print(f\"train_loss = {train_loss} | val_loss = {val_loss} | time = {dur}s\")\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        torch.save(vit_b16_device.state_dict(), vit_b16_path)\n        print(f\">>Saved model's parameters to {vit_b16_path}\")\n    \n    train_loss_s1.append(train_loss)\n    val_loss_s1.append(val_loss)\n\n# ======LOAD BEST & UNFREEZE======\nvit_b16_device.load_state_dict(torch.load(vit_b16_path))\nprint(\"Load model successfully\")\n\nfor params in vit_b16_device.parameters():\n    params.requires_grad = True\n\n# ======STAGE 2: Fine-tune whole model======\nEPOCHS_S2 = 15\ntrain_loss_s2, val_loss_s2 = [], []\n\noptimizer_stage_2 = torch.optim.Adam(vit_b16_device.parameters(), lr=1e-5)\nlr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n    optimizer_stage_2,\n    T_max=EPOCHS_S2,\n    eta_min=1e-7\n)\nprint(\"Setup stage 2 completed.\")\n\nprint(\"======STAGE 2======\")\nfor e in range(EPOCHS_S2):\n    print(f\"EPOCH [{e + 1}/{EPOCHS_S2}]: \", end=\"\")\n\n    start = time.time()\n    train_loss = train_one_epoch(vit_b16_device, train_dataloader, criterion, optimizer_stage_2, lr_scheduler)\n    val_loss = evaluate(vit_b16_device, val_dataloader, criterion)\n    end = time.time()\n\n    dur = round(end - start)\n    print(f\"train_loss = {train_loss} | val_loss = {val_loss} | time = {dur}s\")\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        torch.save(vit_b16_device.state_dict(), vit_b16_path)\n        print(f\">>Saved model's parameters to {vit_b16_path}\")\n\n    train_loss_s2.append(train_loss)\n    val_loss_s2.append(val_loss)\n\nvit_b16_device.load_state_dict(torch.load(vit_b16_path))\n\n# ======STAGE 3: Submission======\nprint(\"=====STAGE 3======\")\nsubmit(vit_b16_device, test_dataloader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-13T16:11:44.794564Z","iopub.execute_input":"2026-05-13T16:11:44.794903Z","iopub.status.idle":"2026-05-13T17:17:05.090202Z","shell.execute_reply.started":"2026-05-13T16:11:44.794866Z","shell.execute_reply":"2026-05-13T17:17:05.088919Z"}},"outputs":[],"execution_count":null}]}