{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-18T09:16:46.784166Z","iopub.execute_input":"2025-06-18T09:16:46.784375Z","iopub.status.idle":"2025-06-18T09:16:49.206064Z","shell.execute_reply.started":"2025-06-18T09:16:46.784357Z","shell.execute_reply":"2025-06-18T09:16:49.205251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 1: Installations\n!pip install -q timm py7zr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T09:24:20.827436Z","iopub.execute_input":"2025-06-18T09:24:20.827725Z","iopub.status.idle":"2025-06-18T09:25:55.782004Z","shell.execute_reply.started":"2025-06-18T09:24:20.827706Z","shell.execute_reply":"2025-06-18T09:25:55.781088Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 2: Imports & Configuration\nimport os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport py7zr\nfrom tqdm.notebook import tqdm\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nimport torchvision.transforms as T\nimport timm\n\n# --- Configuration ---\nclass CFG:\n    # Kaggle environment paths\n    data_path = '/kaggle/input/cifar-10/'\n    train_archive = os.path.join(data_path, 'train.7z')\n    test_archive = os.path.join(data_path, 'test.7z')\n    train_labels_csv = os.path.join(data_path, 'trainLabels.csv')\n    \n    # Output paths\n    extract_path = '/kaggle/working/cifar10/'\n    train_img_path = os.path.join(extract_path, 'train')\n    test_img_path = os.path.join(extract_path, 'test')\n    model_save_path = '/kaggle/working/best_model.pth'\n\n    # Model and Training parameters\n    model_name = 'vit_base_patch16_224.augreg_in21k' # A powerful pre-trained ViT\n    img_size = 224\n    batch_size = 32\n    epochs = 10  # A good starting point for fine-tuning\n    learning_rate = 1e-4\n    weight_decay = 1e-5\n    label_smoothing = 0.1\n    validation_split = 0.1 # Use 10% of training data for validation\n    \n    # Hardware\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    \n    # CIFAR-10 specific\n    num_classes = 10\n    class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\n\nprint(f\"Using device: {CFG.device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T09:25:59.589153Z","iopub.execute_input":"2025-06-18T09:25:59.589441Z","iopub.status.idle":"2025-06-18T09:26:15.166622Z","shell.execute_reply.started":"2025-06-18T09:25:59.589414Z","shell.execute_reply":"2025-06-18T09:26:15.166005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3: Extract Data\nos.makedirs(CFG.extract_path, exist_ok=True)\n\nprint(\"Extracting train.7z...\")\nwith py7zr.SevenZipFile(CFG.train_archive, mode='r') as z:\n    z.extractall(path=CFG.extract_path)\nprint(\"Train images extracted.\")\n\nprint(\"Extracting test.7z...\")\nwith py7zr.SevenZipFile(CFG.test_archive, mode='r') as z:\n    z.extractall(path=CFG.extract_path)\nprint(\"Test images extracted.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T09:26:23.138617Z","iopub.execute_input":"2025-06-18T09:26:23.139328Z","iopub.status.idle":"2025-06-18T09:28:44.562818Z","shell.execute_reply.started":"2025-06-18T09:26:23.139303Z","shell.execute_reply":"2025-06-18T09:28:44.561987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4: Dataset & DataLoaders\n\n# Load labels and create a mapping for faster lookup\ntrain_df = pd.read_csv(CFG.train_labels_csv)\n# Create a dictionary mapping class names to integer indices\nclass_to_idx = {name: i for i, name in enumerate(CFG.class_names)}\n# Map string labels to integer indices in the DataFrame\ntrain_df['label_idx'] = train_df['label'].map(class_to_idx)\n# Create a mapping from image ID to label index for the dataset\nid_to_label = dict(zip(train_df['id'], train_df['label_idx']))\n\n\nclass Cifar10Dataset(Dataset):\n    def __init__(self, img_dir, image_ids, labels_map=None, transform=None):\n        self.img_dir = img_dir\n        self.image_ids = image_ids\n        self.labels_map = labels_map\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        img_path = os.path.join(self.img_dir, f\"{image_id}.png\")\n        image = Image.open(img_path).convert(\"RGB\")\n\n        if self.transform:\n            image = self.transform(image)\n\n        if self.labels_map:\n            label = self.labels_map[image_id]\n            return image, torch.tensor(label, dtype=torch.long)\n        else:\n            return image, image_id\n\n# Define data augmentations and normalization\n# ViT models expect a specific normalization scheme from their ImageNet pre-training\ndata_transforms = {\n    'train': T.Compose([\n        T.Resize((CFG.img_size, CFG.img_size)),\n        T.TrivialAugmentWide(),  # State-of-the-art automatic augmentation\n        T.ToTensor(),\n        T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) # ViT standard normalization\n    ]),\n    'val': T.Compose([\n        T.Resize((CFG.img_size, CFG.img_size)),\n        T.ToTensor(),\n        T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n    ]),\n}\n\n# --- Create Datasets ---\nall_train_ids = train_df['id'].values\nfull_dataset = Cifar10Dataset(CFG.train_img_path, all_train_ids, id_to_label)\n\n# Split into training and validation sets\nval_size = int(len(full_dataset) * CFG.validation_split)\ntrain_size = len(full_dataset) - val_size\ntrain_dataset, val_dataset = random_split(full_dataset, [train_size, val_size])\n\n# Apply the correct transforms to each split\ntrain_dataset.dataset.transform = data_transforms['train']\nval_dataset.dataset.transform = data_transforms['val']\n\n# --- Create DataLoaders ---\ntrain_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=os.cpu_count(), pin_memory=True)\nval_loader = DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=os.cpu_count(), pin_memory=True)\n\nprint(f\"Training samples: {len(train_dataset)}, Validation samples: {len(val_dataset)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T09:28:49.507423Z","iopub.execute_input":"2025-06-18T09:28:49.508158Z","iopub.status.idle":"2025-06-18T09:28:49.599608Z","shell.execute_reply.started":"2025-06-18T09:28:49.508133Z","shell.execute_reply":"2025-06-18T09:28:49.598961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5: Model, Loss, Optimizer Setup\n\n# Load pre-trained ViT model\nmodel = timm.create_model(\n    CFG.model_name,\n    pretrained=True,\n    num_classes=CFG.num_classes\n)\nmodel.to(CFG.device)\n\n# Loss Function with Label Smoothing\n# This regularization prevents the model from becoming overconfident\nloss_fn = nn.CrossEntropyLoss(label_smoothing=CFG.label_smoothing)\n\n# AdamW Optimizer - better than Adam for generalization\noptimizer = torch.optim.AdamW(model.parameters(), lr=CFG.learning_rate, weight_decay=CFG.weight_decay)\n\n# OneCycleLR Scheduler for faster convergence and better performance\nscheduler = torch.optim.lr_scheduler.OneCycleLR(\n    optimizer,\n    max_lr=CFG.learning_rate,\n    steps_per_epoch=len(train_loader),\n    epochs=CFG.epochs\n)\n\n# Gradient Scaler for Automatic Mixed Precision (AMP)\n# This speeds up training significantly on modern GPUs\nscaler = torch.cuda.amp.GradScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T09:29:12.424619Z","iopub.execute_input":"2025-06-18T09:29:12.425144Z","iopub.status.idle":"2025-06-18T09:29:16.008390Z","shell.execute_reply.started":"2025-06-18T09:29:12.425118Z","shell.execute_reply":"2025-06-18T09:29:16.007582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 6: Training & Validation Loop\n\ndef train_one_epoch(model, loader, optimizer, scheduler, loss_fn, scaler, device):\n    model.train()\n    total_loss = 0\n    progress_bar = tqdm(loader, desc=\"Training\", leave=False)\n    \n    for inputs, labels in progress_bar:\n        inputs, labels = inputs.to(device), labels.to(device)\n        \n        # Mixed precision training context\n        with torch.cuda.amp.autocast():\n            outputs = model(inputs)\n            loss = loss_fn(outputs, labels)\n        \n        # Backward pass with scaler\n        optimizer.zero_grad()\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        \n        scheduler.step() # Step scheduler at each batch for OneCycleLR\n        \n        total_loss += loss.item()\n        progress_bar.set_postfix(loss=loss.item(), lr=scheduler.get_last_lr()[0])\n        \n    return total_loss / len(loader)\n\ndef validate_one_epoch(model, loader, loss_fn, device):\n    model.eval()\n    total_loss = 0\n    correct_predictions = 0\n    total_samples = 0\n    \n    with torch.no_grad():\n        progress_bar = tqdm(loader, desc=\"Validating\", leave=False)\n        for inputs, labels in progress_bar:\n            inputs, labels = inputs.to(device), labels.to(device)\n            \n            with torch.cuda.amp.autocast():\n                outputs = model(inputs)\n                loss = loss_fn(outputs, labels)\n\n            total_loss += loss.item()\n            _, preds = torch.max(outputs, 1)\n            correct_predictions += torch.sum(preds == labels.data)\n            total_samples += labels.size(0)\n\n    avg_loss = total_loss / len(loader)\n    accuracy = correct_predictions.double() / total_samples\n    return avg_loss, accuracy\n\n# --- Main Training Loop ---\nbest_val_accuracy = 0.0\n\nfor epoch in range(CFG.epochs):\n    print(f\"--- Epoch {epoch+1}/{CFG.epochs} ---\")\n    \n    train_loss = train_one_epoch(model, train_loader, optimizer, scheduler, loss_fn, scaler, CFG.device)\n    val_loss, val_accuracy = validate_one_epoch(model, val_loader, loss_fn, CFG.device)\n    \n    print(f\"Epoch {epoch+1}: Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Accuracy: {val_accuracy:.4f}\")\n    \n    if val_accuracy > best_val_accuracy:\n        best_val_accuracy = val_accuracy\n        print(f\"New best model found! Saving to {CFG.model_save_path}\")\n        torch.save(model.state_dict(), CFG.model_save_path)\n\nprint(\"\\n--- Training Finished ---\")\nprint(f\"Best Validation Accuracy: {best_val_accuracy:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T09:29:45.545767Z","iopub.execute_input":"2025-06-18T09:29:45.546283Z","iopub.status.idle":"2025-06-18T10:36:38.114803Z","shell.execute_reply.started":"2025-06-18T09:29:45.546261Z","shell.execute_reply":"2025-06-18T10:36:38.114020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 7: Inference and Submission\n\n# Load the best model weights\nmodel.load_state_dict(torch.load(CFG.model_save_path))\nmodel.to(CFG.device)\nmodel.eval()\n\n# Prepare the test dataset\ntest_image_files = sorted([f for f in os.listdir(CFG.test_img_path) if f.endswith('.png')])\ntest_ids = [int(os.path.splitext(f)[0]) for f in test_image_files]\n\ntest_dataset = Cifar10Dataset(\n    img_dir=CFG.test_img_path,\n    image_ids=test_ids,\n    labels_map=None,  # No labels for test set\n    transform=data_transforms['val'] # Use validation transforms for test\n)\n\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=os.cpu_count())\n\n# --- Generate Predictions ---\npredictions = []\nwith torch.no_grad():\n    progress_bar = tqdm(test_loader, desc=\"Predicting on Test Set\")\n    for inputs, image_ids in progress_bar:\n        inputs = inputs.to(CFG.device)\n        \n        with torch.cuda.amp.autocast():\n            outputs = model(inputs)\n            \n        _, preds = torch.max(outputs, 1)\n        \n        for i, pred_idx in enumerate(preds):\n            image_id = image_ids[i].item()\n            label_name = CFG.class_names[pred_idx.item()]\n            predictions.append((image_id, label_name))\n\n# --- Create Submission File ---\nsubmission_df = pd.DataFrame(predictions, columns=['id', 'label'])\nsubmission_df = submission_df.sort_values(by='id')\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(\"\\nSubmission file created successfully!\")\nprint(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-18T11:25:15.359723Z","iopub.execute_input":"2025-06-18T11:25:15.360047Z","iopub.status.idle":"2025-06-18T11:39:09.349520Z","shell.execute_reply.started":"2025-06-18T11:25:15.359989Z","shell.execute_reply":"2025-06-18T11:39:09.348785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}