{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T05:55:54.756544Z","iopub.execute_input":"2026-06-01T05:55:54.756866Z","iopub.status.idle":"2026-06-01T05:56:17.577369Z","shell.execute_reply.started":"2026-06-01T05:55:54.756830Z","shell.execute_reply":"2026-06-01T05:56:17.576515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch,time, copy\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision\nimport torchvision.transforms as transforms\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom random import random\nfrom collections import OrderedDict\nfrom typing import Any, Callable, List, Optional, Type, Union\nfrom torchvision.models import mobilenet_v3_small, MobileNet_V3_Small_Weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T05:56:25.399047Z","iopub.execute_input":"2026-06-01T05:56:25.399966Z","iopub.status.idle":"2026-06-01T05:56:31.323276Z","shell.execute_reply.started":"2026-06-01T05:56:25.399915Z","shell.execute_reply":"2026-06-01T05:56:31.322267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\nimport shutil\nfrom pathlib import Path\n\nroot = Path('./data')\ninput_path = Path('../input/cifar-10')\n\nwith py7zr.SevenZipFile(input_path/'train.7z', mode='r') as z:\n    z.extractall(root)\n\nwith py7zr.SevenZipFile(input_path/'test.7z', mode='r') as z:\n    z.extractall(root)\n\nshutil.copy(input_path/'trainLabels.csv', root/'trainLabels.csv')\n\n(root/'train').rename(root/'original_train')\n(root/'test').rename(root/'original_test')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T05:56:34.228669Z","iopub.execute_input":"2026-06-01T05:56:34.229715Z","iopub.status.idle":"2026-06-01T05:59:15.836487Z","shell.execute_reply.started":"2026-06-01T05:56:34.229684Z","shell.execute_reply":"2026-06-01T05:59:15.835532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def copy_file(source_directory, destination_directory, filename):\n    \"\"\"\n    Utility function used to copy a file from a source_directory to a destination_directory\n    \"\"\"\n    destination_directory.mkdir(parents=True, exist_ok=True)\n    shutil.copy(source_directory/filename, destination_directory/filename)\n    \ndef organize_train_valid_dataset(root, labels, valid_probability=0.1):\n    \"\"\"\n    Creates the train, train_valid and valid folders respecting PyTorch's ImageDataset structure, performing\n    train/validation split based on the given percentage\n    \"\"\"\n    source_directory = root/'original_train'\n    \n    with os.scandir(source_directory) as it:\n        for entry in it:\n            if entry.is_file():\n                img_index = entry.name.split('.')[0]  # The index is the name of the image except the extension\n                img_class = labels[labels.id==int(img_index)].label.values[0]  # Find the class by looking up the index in the DF\n                \n                # Randomly assign the image to the valid dataset with probability 'valid_probability'\n                channel = Path('train') if random()>valid_probability else Path('valid')\n                destination_directory = root/channel/img_class\n                \n                # Copy the image to either the train or valid folder, and also to the train_valid folder\n                copy_file(source_directory, destination_directory, entry.name)\n                copy_file(source_directory, root/'train_valid'/img_class, entry.name)\n\ndef organize_test_dataset(root):\n    \"\"\"\n    Creates the test folder respecting PyTorch's ImageDataset structure, using a dummy 'undefined' label\n    \"\"\"\n    source_directory = root/'original_test'\n        \n    with os.scandir(source_directory) as it:\n        for entry in it:\n            if entry.is_file():\n                img_index = entry.name.split('.')[0]  # The index is the name of the image except the extension\n\n                channel = Path('test')\n                destination_directory = root/channel/'undefined'\n\n                copy_file(source_directory, destination_directory, entry.name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:00:16.623126Z","iopub.execute_input":"2026-06-01T06:00:16.624114Z","iopub.status.idle":"2026-06-01T06:00:16.632234Z","shell.execute_reply.started":"2026-06-01T06:00:16.624083Z","shell.execute_reply":"2026-06-01T06:00:16.631299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels = pd.read_csv(root/'trainLabels.csv')\n\n# Create the train/train_valid/valid folder structure\nvalid_probability = 0.1\norganize_train_valid_dataset(root, labels, valid_probability)\n\n# Create the test folder structure\norganize_test_dataset(root)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:00:19.666934Z","iopub.execute_input":"2026-06-01T06:00:19.667650Z","iopub.status.idle":"2026-06-01T06:01:47.966332Z","shell.execute_reply.started":"2026-06-01T06:00:19.667621Z","shell.execute_reply":"2026-06-01T06:01:47.965196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"classes = [\"airplane\",\"automobile\",\"bird\",\"cat\",\"deer\",\"dog\",\"frog\",\"horse\",\"ship\",\"truck\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:03:26.243718Z","iopub.execute_input":"2026-06-01T06:03:26.244215Z","iopub.status.idle":"2026-06-01T06:03:26.249472Z","shell.execute_reply.started":"2026-06-01T06:03:26.244185Z","shell.execute_reply":"2026-06-01T06:03:26.248558Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Calculate transform mean and std","metadata":{}},{"cell_type":"code","source":"train_dataset = torchvision.datasets.ImageFolder(\n    root/'train', \n    transform=torchvision.transforms.Compose([\n        # Resize step is required as we will use a ResNet model, which accepts at leats 224x224 images\n        torchvision.transforms.Resize((224,224)),  \n        torchvision.transforms.ToTensor(),\n    ])\n)\n\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=False, num_workers=2, pin_memory=True)\n\nmeans = []\nstdevs = []\nfor X, _ in train_dataloader:\n    # Dimensions 0,2,3 are respectively the batch, height and width dimensions\n    means.append(X.mean(dim=(0,2,3)))\n    stdevs.append(X.std(dim=(0,2,3)))\n\nmean = torch.stack(means, dim=0).mean(dim=0)\nstdev = torch.stack(stdevs, dim=0).mean(dim=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:03:26.860413Z","iopub.execute_input":"2026-06-01T06:03:26.861118Z","iopub.status.idle":"2026-06-01T06:04:25.896574Z","shell.execute_reply.started":"2026-06-01T06:03:26.861088Z","shell.execute_reply":"2026-06-01T06:04:25.895194Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"IMAGE_SIZE = 224\nbatch_size = 64\n\ntrain_transforms = torchvision.transforms.Compose([\n        torchvision.transforms.Resize((224,224)),\n        torchvision.transforms.AutoAugment(policy=torchvision.transforms.AutoAugmentPolicy.CIFAR10),\n        torchvision.transforms.ToTensor(),\n        torchvision.transforms.Normalize(mean, stdev)\n    ])\n\ntrain_dataset, train_valid_dataset = [torchvision.datasets.ImageFolder(folder, transform=train_transforms) for folder in [root/'train', root/'train_valid']]\n\n\nvalid_transforms = torchvision.transforms.Compose([\n        torchvision.transforms.Resize((224,224)),\n        torchvision.transforms.ToTensor(),\n        torchvision.transforms.Normalize(mean, stdev)\n    ])\n\nvalid_dataset, test_dataset = [torchvision.datasets.ImageFolder(folder, transform=valid_transforms) for folder in [root/'valid', root/'test']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:04:25.899072Z","iopub.execute_input":"2026-06-01T06:04:25.899477Z","iopub.status.idle":"2026-06-01T06:04:27.075675Z","shell.execute_reply.started":"2026-06-01T06:04:25.899453Z","shell.execute_reply":"2026-06-01T06:04:27.074941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_gpus = torch.cuda.device_count()\n\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=256, shuffle=True, num_workers=2*num_gpus, pin_memory=True)\ntrain_valid_dataloader = torch.utils.data.DataLoader(train_valid_dataset, batch_size=256, shuffle=True, num_workers=2*num_gpus, pin_memory=True)\n\nvalid_dataloader = torch.utils.data.DataLoader(valid_dataset, batch_size=512, shuffle=False, num_workers=2*num_gpus, pin_memory=True)\ntest_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=512, shuffle=False, num_workers=2*num_gpus, pin_memory=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:04:27.077212Z","iopub.execute_input":"2026-06-01T06:04:27.077580Z","iopub.status.idle":"2026-06-01T06:04:27.089916Z","shell.execute_reply.started":"2026-06-01T06:04:27.077548Z","shell.execute_reply":"2026-06-01T06:04:27.088831Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_dataloader.dataset)","metadata":{"execution":{"iopub.status.busy":"2026-06-01T06:04:27.091964Z","iopub.execute_input":"2026-06-01T06:04:27.092306Z","iopub.status.idle":"2026-06-01T06:04:27.103796Z","shell.execute_reply.started":"2026-06-01T06:04:27.092285Z","shell.execute_reply":"2026-06-01T06:04:27.102944Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def picshow(pic,labels):\n    labels = [classes[labels[j]] for j in range(8)]\n    pic = pic * 0.25 + 0.5\n    plt.figure(figsize = (32,4),clear = True)\n    plt.imshow(np.transpose(pic.numpy(), (1,2,0)))\n    plt.xticks(np.linspace(16.5*(IMAGE_SIZE//32),255*(IMAGE_SIZE//32),8),labels, fontsize = 28)\n    plt.show()\n    \ndataiter = iter(train_dataloader)\npic, labels = next(dataiter)\n\nprint(pic.size())\npicshow(torchvision.utils.make_grid(pic[0:8]),labels)","metadata":{"execution":{"iopub.status.busy":"2026-06-01T06:04:27.104998Z","iopub.execute_input":"2026-06-01T06:04:27.105574Z","iopub.status.idle":"2026-06-01T06:04:28.992799Z","shell.execute_reply.started":"2026-06-01T06:04:27.105528Z","shell.execute_reply":"2026-06-01T06:04:28.991969Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load and initialized model","metadata":{}},{"cell_type":"code","source":"def get_model():\n    model = mobilenet_v3_small( weights = MobileNet_V3_Small_Weights.IMAGENET1K_V1)\n    model.classifier[3] = nn.Linear(1024, 10, bias = True)\n    for m in model.classifier.modules():\n        if isinstance(m, nn.Linear):\n            nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='relu')\n            if m.bias is not None:\n                nn.init.zeros_(m.bias)\n    print(\"Tuning the model and initialized completed!\")\n    return model","metadata":{"execution":{"iopub.status.busy":"2026-06-01T06:04:28.994271Z","iopub.execute_input":"2026-06-01T06:04:28.994598Z","iopub.status.idle":"2026-06-01T06:04:29.004007Z","shell.execute_reply.started":"2026-06-01T06:04:28.994552Z","shell.execute_reply":"2026-06-01T06:04:29.002954Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build a trainer","metadata":{}},{"cell_type":"code","source":"class Trainer():\n    def __init__(self, model, train_loader, valid_loader, criterion, \n                 optimizer, num_epoch, lr_scheduler  = None, early_stop = 5, device = None):\n        self.train_loader = train_loader\n        self.valid_loader = valid_loader\n        self.criterion = criterion\n        self.optim = optimizer\n        self.num_epoch = num_epoch  \n        self.lr_scheduler = lr_scheduler \n        self.early_stop = early_stop\n\n        self.device = device\n        self.model = model.to(device)\n        self.best_loss = float('inf')\n\n        self.stats = {\"Epoch\": 0, \"train_loss\": [], \"valid_loss\": [],\n                      \"valid_acc\": []}\n        \n    def _epoch(self, x, y, istraining = True):\n        if istraining:\n            self.optim.zero_grad()\n        with torch.set_grad_enabled(istraining):\n            outs = self.model(x)\n            loss = self.criterion(outs, y)\n            if istraining:\n                loss.backward()\n                self.optim.step()\n        return loss.item(), outs\n\n    def train(self):\n        losses = 0.0\n        accuracy = 0.0\n        samples = 0\n        self.model.train()\n        for x, y in tqdm(self.train_loader):\n            x = x.to(self.device)\n            y = y.to(self.device)\n            batch_size = x.shape[0]\n            loss, preds = self._epoch(x, y, True)\n            losses += loss * batch_size\n            accuracy += (torch.argmax(preds, dim=1) == y).sum().item()\n            samples += batch_size\n        return losses/samples, accuracy/samples\n\n    def eval(self):\n        losses = 0.0\n        accuracy = 0.0\n        samples = 0\n        self.model.eval()\n        with torch.no_grad():\n            for x, y in tqdm(self.valid_loader):\n                x = x.to(self.device)\n                y = y.to(self.device)\n                batch_size = x.shape[0]\n                loss, preds = self._epoch(x, y, False)\n                losses += loss * batch_size\n                accuracy += (torch.argmax(preds, dim=1) == y).sum().item()\n                samples += batch_size\n        return losses/samples, accuracy/samples\n\n    def save(self, savepath):\n        \"\"\"\n        No need for phased training, No save the optimizer or lr_scheduler\n        \"\"\"\n        torch.save(self.model.state_dict(), savepath)\n    \n    def run(self):\n        n_count = 0\n        for epoch in range(self.num_epoch):\n            self.stats[\"Epoch\"] = epoch+1\n            print(f\"Epoch: {epoch+1}\")\n            train_loss, _ = self.train()\n            print(f\"train loss: {train_loss:.2f}\")\n            self.stats[\"train_loss\"].append(train_loss)\n            \n            if self.valid_loader is not None:\n                valid_loss, valid_acc = self.eval()\n                self.stats[\"valid_loss\"].append(valid_loss)\n                self.stats[\"valid_acc\"].append(valid_acc)\n                print(f\"valid loss: {valid_loss:.2f}, acc: {valid_acc*100:.2f}%\")\n                if valid_loss < self.best_loss:\n                    self.best_loss = valid_loss\n                    n_count = 0\n                    # self.save(f\"model_dict_{epoch+1}.pth\")\n                else:\n                    n_count += 1\n                if n_count >= self.early_stop:\n                    print(f\"Epoch is {epoch+1} early stopping ......\")\n                    break\n\n            if self.lr_scheduler is not None:\n                self.lr_scheduler.step()\n        return None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:06:27.902636Z","iopub.execute_input":"2026-06-01T06:06:27.903016Z","iopub.status.idle":"2026-06-01T06:06:27.916842Z","shell.execute_reply.started":"2026-06-01T06:06:27.902988Z","shell.execute_reply":"2026-06-01T06:06:27.915949Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tiered training","metadata":{}},{"cell_type":"code","source":"model = get_model()\nnum_epoch = 20\nlr, weight_decay = 1e-5, 5e-4\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nbody_params = [param for name, param in model.named_parameters() if 'classifier' not in str(name)]\nfc_params = model.classifier.parameters()\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam([{'params': body_params}, {'params': fc_params, 'lr': lr*10}], lr = lr, weight_decay = weight_decay)\ntrainer = Trainer(model, train_dataloader, valid_dataloader, criterion, optimizer, num_epoch = num_epoch, device = device)","metadata":{"execution":{"iopub.status.busy":"2026-06-01T06:06:30.026702Z","iopub.execute_input":"2026-06-01T06:06:30.027200Z","iopub.status.idle":"2026-06-01T06:06:30.135502Z","shell.execute_reply.started":"2026-06-01T06:06:30.027169Z","shell.execute_reply":"2026-06-01T06:06:30.134728Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer.run()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:06:34.085549Z","iopub.execute_input":"2026-06-01T06:06:34.086451Z","iopub.status.idle":"2026-06-01T06:35:12.712070Z","shell.execute_reply.started":"2026-06-01T06:06:34.086422Z","shell.execute_reply":"2026-06-01T06:35:12.710443Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Full training","metadata":{}},{"cell_type":"code","source":"model = get_model()\nlr, weight_decay, epochs = 1e-5, 5e-4, 30\ncriterion = torch.nn.CrossEntropyLoss()\n\nbody_params = [param for name, param in model.named_parameters() if 'classifier' not in str(name)]\nfc_params = model.classifier.parameters()\noptimizer = torch.optim.Adam([{'params':body_params}, {'params': fc_params, 'lr': lr*10}], lr=lr, weight_decay=weight_decay)\n\nfull_trainer = Trainer(model, train_valid_dataloader, None, criterion, optimizer, num_epoch = epochs, device = device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:42:55.357038Z","iopub.execute_input":"2026-06-01T06:42:55.357756Z","iopub.status.idle":"2026-06-01T06:42:55.460183Z","shell.execute_reply.started":"2026-06-01T06:42:55.357705Z","shell.execute_reply":"2026-06-01T06:42:55.459294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_trainer.run()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T06:43:40.325461Z","iopub.execute_input":"2026-06-01T06:43:40.325784Z","iopub.status.idle":"2026-06-01T07:26:19.397713Z","shell.execute_reply.started":"2026-06-01T06:43:40.325762Z","shell.execute_reply":"2026-06-01T07:26:19.396675Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = []\n\nfull_trainer.model.eval()\nwith torch.no_grad():\n    for X, _ in tqdm(test_dataloader):\n        X = X.to(device)\n        preds.extend(full_trainer.model(X).argmax(dim=1).type(torch.int32).cpu().numpy())\n\nids = list(range(1, len(test_dataset)+1))\nids.sort(key=lambda x: str(x))\n\ndf = pd.DataFrame({'id': ids, 'label': preds})\ndf['label'] = df['label'].apply(lambda x: train_dataset.classes[x])\ndf.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T07:36:37.885131Z","iopub.execute_input":"2026-06-01T07:36:37.885935Z","iopub.status.idle":"2026-06-01T07:42:58.012556Z","shell.execute_reply.started":"2026-06-01T07:36:37.885891Z","shell.execute_reply":"2026-06-01T07:42:58.011279Z"}},"outputs":[],"execution_count":null}]}