{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"},{"sourceId":12583723,"sourceType":"datasetVersion","datasetId":7859567}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Notes\n\n- Mạng thông dụng nhất hiện tại thường đều sử dụng cấu trúc Residual Network (ResNet), nên bài này sử dụng một custom ResNet, có thể mọi người tự thay đổi bằng ResNet-18.\n- Một mạng CNN thông thường sẽ là CNN (extractor) + FC. Tùy thuộc vào lương data, đôi khi quá nhiều lớp FC sẽ gây ra hiện tượng overfit (tập train hội tụ về accuracy ~ 1 quá sớm nhưng validation chưa kịp theo). Với mỗi qua một lớp layers sẽ là 1 lần đạo hàm backward (1 epoch train = 1 forward để tạo ra y_predict và dùng loss để backward cập nhật trọng số bằng đạo hàm), mà đạo hàm f(x) liên tục sẽ tiến về 0 (x^2 -> 2x -> 2 -> 0), tạo nên hiện tượng Vanishing Gradient\n- Learning Rate sẽ ảnh hưởng đến tốc độ hội tụ, trong trường hợp lr lớn sẽ học nhanh, nhưng đến phần gần hội tụ (lõm) thì do lr quá lớn sẽ ko về điểm hội tụ (toàn cục) đc, vì vậy có cơ chế Learning Rate Scheduler để tự điều chỉnh lr trong quá trình train model\n- Đối với cấu trúc ResNet-18 thông thường, lớp Conv2D đầu tiên sẽ là 7x7 stride 2 + maxPool, nhưng vì cifar size là 32x32 nên resolution ảnh sẽ drop khá cao sau khi qua kernel 7x7 (sẽ là 8x8), vậy nên có thể giảm kernel size xuống 3x3. (Cứ mỗi qua kernel thì shape ban đầu sẽ giảm đi nếu không có padding, còn số depth sẽ phụ thuộc vào số kernel trong layers conv2D đó)\n- Một cách để tránh overfitting là DropOut, là bỏ một số trọng số trong layer đc apply, có thể là lớp cuối cùng của CNN (extractor)\n- Initialization từng layer model cũng sẽ ảnh hưởng đến trong số ban đầu của model, trọng số đc khởi tạo tốt thì cập nhât đạo hàm cũng tốt (vd: có thể khởi tạo kernel 3x3 là [[000][000][000]], hoặc [[111][111][111]], nhưng nên theo 1 cơ chế nhất đinh nào đó)\n- Batch size càng lớn sẽ học càng nhanh, nhưng có một vấn đề liên quan đến BatchNorm, batch_size càng lớn thì gradient cập nhật càng ít (mọi người tự tìm thiểu thêm nhé, em ko rõ :(( )\n\n- Quan trọng nhất: DATA Augmentation. Data càng chất lượng và nhiều điều kiện khác nhau thì model càng generalization.","metadata":{}},{"cell_type":"code","source":"ls /kaggle/input/cifar-10","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.450941Z","iopub.execute_input":"2025-07-31T03:41:02.451619Z","iopub.status.idle":"2025-07-31T03:41:02.590842Z","shell.execute_reply.started":"2025-07-31T03:41:02.451595Z","shell.execute_reply":"2025-07-31T03:41:02.589922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !7z x /kaggle/input/cifar-10/train.7z -o/kaggle/working/train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.592683Z","iopub.execute_input":"2025-07-31T03:41:02.592900Z","iopub.status.idle":"2025-07-31T03:41:02.596464Z","shell.execute_reply.started":"2025-07-31T03:41:02.592879Z","shell.execute_reply":"2025-07-31T03:41:02.595776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport torch\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nimport random\n\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader, random_split, Subset\nimport torch.optim as optim\nfrom tqdm import tqdm\nimport torchvision.datasets as dsets\nfrom torchvision.datasets import ImageFolder\nimport torchvision.transforms as transforms\nfrom sklearn.model_selection import train_test_split\n\nprint(torch.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.597331Z","iopub.execute_input":"2025-07-31T03:41:02.598054Z","iopub.status.idle":"2025-07-31T03:41:02.609962Z","shell.execute_reply.started":"2025-07-31T03:41:02.598030Z","shell.execute_reply":"2025-07-31T03:41:02.609397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(42)\ndevice","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.611759Z","iopub.execute_input":"2025-07-31T03:41:02.611972Z","iopub.status.idle":"2025-07-31T03:41:02.623604Z","shell.execute_reply.started":"2025-07-31T03:41:02.611957Z","shell.execute_reply":"2025-07-31T03:41:02.622974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CLASS_TO_IDX = {\n    'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4,\n    'dog': 5,      'frog': 6,       'horse': 7, 'ship': 8, 'truck': 9\n}\n\nIDX_TO_CLASS = {v: k for k,v in CLASS_TO_IDX.items()}\n\ncomposed_transform = transforms.Compose([\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.ConvertImageDtype(torch.float),\n    transforms.Normalize(mean=[0.4914, 0.4822, 0.4465] , std=[0.247, 0.243, 0.261]), \n    transforms.RandomErasing(),\n])\n\nval_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.ConvertImageDtype(torch.float),\n    transforms.Normalize(mean=[0.4914, 0.4822, 0.4465] , std=[0.247, 0.243, 0.261]), \n])\n\n\nclass CustomImageDataset(Dataset):\n    def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):\n        self.img_labels = pd.read_csv(annotations_file)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, str(self.img_labels.iloc[idx, 0]) + '.png')\n        image = Image.open(img_path).convert(\"RGB\")\n        label = self.img_labels.iloc[idx, 1]\n        label = CLASS_TO_IDX[label]\n        if self.transform:\n            image = self.transform(image)\n        if self.target_transform:\n            label = self.target_transform(label)\n        return image, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.624238Z","iopub.execute_input":"2025-07-31T03:41:02.624502Z","iopub.status.idle":"2025-07-31T03:41:02.634073Z","shell.execute_reply.started":"2025-07-31T03:41:02.624480Z","shell.execute_reply":"2025-07-31T03:41:02.633283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cifar = CustomImageDataset(\n#     annotations_file='/kaggle/input/cifar10-7z-extraction/trainLabels.csv', \n#     img_dir='/kaggle/input/cifar10-7z-extraction/train', \n#     transform=composed_transform\n# )\n\n# train_dataset =  dsets.CIFAR10(root='./data', train=True, download=True, transform = composed_transform)\n# validation_dataset = dsets.CIFAR10(root='./data', train=False, download=True, transform = val_transform)\n\n# plt.imshow(train_dataset[0][0].permute(1,2,0))\n# print(IDX_TO_CLASS[train_dataset[0][1]])\n\n# train_loader = DataLoader(dataset=train_dataset, batch_size=256, shuffle=True)\n# val_loader = DataLoader(dataset=validation_dataset, batch_size=256, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.634738Z","iopub.execute_input":"2025-07-31T03:41:02.634937Z","iopub.status.idle":"2025-07-31T03:41:02.643781Z","shell.execute_reply.started":"2025-07-31T03:41:02.634923Z","shell.execute_reply":"2025-07-31T03:41:02.643178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cifar_size = len(os.listdir('/kaggle/input/cifar10-7z-extraction/train'))\nall_indices = np.arange(cifar_size)\ntrain_idx, val_idx = train_test_split(all_indices, train_size=0.8, random_state=42, shuffle=True)\n\ntrain_subset = Subset(\n    CustomImageDataset(\n        annotations_file='/kaggle/input/cifar10-7z-extraction/trainLabels.csv', \n        img_dir='/kaggle/input/cifar10-7z-extraction/train', \n        transform=composed_transform\n    ),\n    train_idx\n)\nval_subset = Subset(\n    CustomImageDataset(\n        annotations_file='/kaggle/input/cifar10-7z-extraction/trainLabels.csv', \n        img_dir='/kaggle/input/cifar10-7z-extraction/train', \n        transform=val_transform\n    ),\n    val_idx\n)\n\ntrain_loader = DataLoader(train_subset, batch_size=256, shuffle=True)\nval_loader = DataLoader(val_subset, batch_size=256, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.644457Z","iopub.execute_input":"2025-07-31T03:41:02.644727Z","iopub.status.idle":"2025-07-31T03:41:02.701650Z","shell.execute_reply.started":"2025-07-31T03:41:02.644703Z","shell.execute_reply":"2025-07-31T03:41:02.701116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(train_subset[0][0].permute(1,2,0))\nprint(IDX_TO_CLASS[train_subset[0][1]])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.702252Z","iopub.execute_input":"2025-07-31T03:41:02.702460Z","iopub.status.idle":"2025-07-31T03:41:02.850016Z","shell.execute_reply.started":"2025-07-31T03:41:02.702444Z","shell.execute_reply":"2025-07-31T03:41:02.849307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ResNet(nn.Module):\n    def __init__(self, in_channels=1, out_channels=1, stride=1):\n        super(ResNet, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, stride=stride, bias=False)\n        self.bnorm1 = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU()\n        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, bias=False)\n        self.bnorm2 = nn.BatchNorm2d(out_channels)\n        \n        if in_channels != out_channels or stride != 1:\n            self.skip = nn.Conv2d(in_channels, out_channels, 1, stride=stride, bias=False)\n        else:\n            self.skip = nn.Identity()\n    \n    def forward(self, x):\n        residual = x\n        x = self.relu(self.bnorm1(self.conv1(x)))\n        x = self.bnorm2(self.conv2(x))\n        # if self.conv1x1:\n        #     residual = self.conv1x1(residual)\n        return self.relu(x + self.skip(residual))\n    \n\nclass Model(nn.Module):\n    def __init__(self, num_classes=10):\n        super(Model, self).__init__()\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bnrom = nn.BatchNorm2d(64)\n        self.maxPool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n        self.res_block1a = ResNet(64, 64)\n        self.res_block1b = ResNet(64, 64)\n\n        self.res_block2a = ResNet(64, 128, stride=1)\n        self.res_block2b = ResNet(128, 128)\n\n        self.res_block3a = ResNet(128, 256, stride=1)\n        self.res_block3b = ResNet(256, 256)\n\n        self.res_block4a = ResNet(256, 512, stride=1)\n        self.res_block4b = ResNet(512, 512)\n\n        self.global_AvgPool = nn.AdaptiveAvgPool2d((1,1))\n        # self.fc1 = nn.Linear(512, 1024)\n        # self.fc2 = nn.Linear(1024, 1024)\n        # self.fc3 = nn.Linear(1024, 512)\n        self.fc4 = nn.Linear(512, num_classes) # Most modern ResNet‑style models do this: GlobalAvgPool → Linear(512→10)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.3)\n\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                # Suitable initialization for ReLU\n                nn.init.normal_(m.weight, mean=0.0, std=0.05)\n                nn.init.constant_(m.bias, 0.0)\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n                if m.bias is not None:\n                    nn.init.constant_(m.bias, 0)\n\n    def forward(self, x):\n        x = self.relu(self.bnrom(self.conv1(x)))\n        x = self.res_block1b(self.res_block1a(x))\n        x = self.res_block2b(self.maxPool(self.res_block2a(x)))\n        x = self.res_block3b(self.maxPool(self.res_block3a(x)))\n        x = self.res_block4b(self.maxPool(self.res_block4a(x)))\n\n        # use adaptive_avg_pool2d 1.1k to achieve global average pooling, \n        # just set the output size to (1, 1)\n        x = self.global_AvgPool(x)\n        x = torch.flatten(x, 1)\n        # x = self.relu(self.fc1(x))\n        # x = self.relu(self.fc2(x))\n        x = self.fc4(self.dropout(x))\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.851668Z","iopub.execute_input":"2025-07-31T03:41:02.851840Z","iopub.status.idle":"2025-07-31T03:41:02.863460Z","shell.execute_reply.started":"2025-07-31T03:41:02.851827Z","shell.execute_reply":"2025-07-31T03:41:02.862763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lr = 3e-4\n\nmodel = Model().to(device)\n\ncriterion = nn.CrossEntropyLoss(label_smoothing=0.1)\noptimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-4)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:02.864074Z","iopub.execute_input":"2025-07-31T03:41:02.864255Z","iopub.status.idle":"2025-07-31T03:41:03.255423Z","shell.execute_reply.started":"2025-07-31T03:41:02.864242Z","shell.execute_reply":"2025-07-31T03:41:03.254830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"epochs = 100\ntrain_loss_lst = []\ntrain_acc_lst = []\nval_loss_lst = []\nval_acc_lst = []\n\ndef freeze_bn(m):\n    if isinstance(m, nn.BatchNorm2d):\n        m.eval()\n        for p in m.parameters():\n            p.requires_grad = False\n\n\nfor epoch in tqdm(range(epochs)):\n    train_loss = 0.0\n    train_acc = 0.0\n    count = 0\n\n    model.train()\n    if epoch == 5:\n        model.apply(freeze_bn)\n        \n    for X_train, y_train in train_loader:\n        X_train, y_train = X_train.float().to(device), y_train.to(device)\n        optimizer.zero_grad()\n        outputs = model(X_train)\n        loss = criterion(outputs, y_train)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n        train_acc += (torch.argmax(outputs, 1) == y_train).sum().item()\n        count += len(y_train)\n    scheduler.step()\n\n    train_loss /= len(train_loader)\n    train_loss_lst.append(train_loss)\n    train_acc /= count\n    train_acc_lst.append(train_acc)\n\n    val_loss = 0.0\n    val_acc = 0.0\n    count = 0\n    model.eval()\n    with torch.no_grad():\n        for X_val, y_val in val_loader:\n            X_val, y_val = X_val.float().to(device), y_val.to(device)\n            outputs = model(X_val)\n            loss = criterion(outputs, y_val)\n            val_loss += loss.item()\n            val_acc += (torch.argmax(outputs, 1) == y_val).sum().item()\n            count += len(y_val)\n\n    val_loss /= len(val_loader)\n    val_loss_lst.append(val_loss)\n    val_acc /= count\n    val_acc_lst.append(val_acc)\n\n    print(f\"EPOCH {epoch+1}/{epochs}, Train_Loss: {train_loss:.4f}, Train_Acc: {train_acc:.4f}, Validation Loss: {val_loss:.4f}, Val_Acc: {val_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:41:03.256110Z","iopub.execute_input":"2025-07-31T03:41:03.256372Z","iopub.status.idle":"2025-07-31T03:47:41.451548Z","shell.execute_reply.started":"2025-07-31T03:41:03.256334Z","shell.execute_reply":"2025-07-31T03:47:41.450527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 2, figsize=(12, 10))\nax[0, 0].plot(train_loss_lst, color='green')\nax[0, 0].set(xlabel='Epoch', ylabel='Loss')\nax[0, 0].set_title('Training Loss')\n\nax[0, 1].plot(val_loss_lst, color='orange')\nax[0, 1].set(xlabel='Epoch', ylabel='Loss')\nax[0, 1].set_title('Validation Loss')\n\nax[1, 0].plot(train_acc_lst, color='green')\nax[1, 0].set(xlabel='Epoch', ylabel='Accuracy')\nax[1, 0].set_title('Training Accuracy')\n\nax[1, 1].plot(val_acc_lst, color='orange')\nax[1, 1].set(xlabel='Epoch', ylabel='Accuracy')\nax[1, 1].set_title('Validation Accuracy')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:47:41.451951Z","iopub.status.idle":"2025-07-31T03:47:41.452160Z","shell.execute_reply.started":"2025-07-31T03:47:41.452060Z","shell.execute_reply":"2025-07-31T03:47:41.452069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, annotations_file, img_dir, transform=None):\n        self.img_labels = annotations_file\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, str(self.img_labels[idx]))\n        image = Image.open(img_path).convert(\"RGB\")\n        if self.transform:\n            image = self.transform(image)\n        return image, 0\n\n\nsample = pd.read_csv('/kaggle/input/cifar10-7z-extraction/sampleSubmission.csv')\ntest_files = sample[\"id\"].astype(str) + \".png\"\ncifar_test = TestDataset(\n    annotations_file=test_files.tolist(), \n    img_dir='/kaggle/input/cifar10-7z-extraction/test', \n    transform=val_transform\n)\ntest_loader = DataLoader(cifar_test, batch_size=256, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:54:32.482840Z","iopub.execute_input":"2025-07-31T03:54:32.483079Z","iopub.status.idle":"2025-07-31T03:54:32.641218Z","shell.execute_reply.started":"2025-07-31T03:54:32.483063Z","shell.execute_reply":"2025-07-31T03:54:32.640644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = []\n\nmodel.eval()\nwith torch.no_grad():\n    for X_test, _ in test_loader:\n        X_test = X_test.to(device)\n        labels = model(X_test).argmax(dim=1).type(torch.int32).cpu().tolist()\n        preds.extend(labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:54:33.020312Z","iopub.execute_input":"2025-07-31T03:54:33.020560Z","iopub.status.idle":"2025-07-31T03:54:34.318146Z","shell.execute_reply.started":"2025-07-31T03:54:33.020543Z","shell.execute_reply":"2025-07-31T03:54:34.317610Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample[\"label\"] = [IDX_TO_CLASS[prediction] for prediction in preds]\nsample.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-31T03:55:39.937447Z","iopub.execute_input":"2025-07-31T03:55:39.937758Z","iopub.status.idle":"2025-07-31T03:55:39.956442Z","shell.execute_reply.started":"2025-07-31T03:55:39.937735Z","shell.execute_reply":"2025-07-31T03:55:39.955323Z"}},"outputs":[],"execution_count":null}]}