{"cells":[{"cell_type":"markdown","id":"1cd391ee","metadata":{},"source":"# EfficientNet-B7 Transfer Learning for Dog Breed Identification\n\nThis notebook implements transfer learning using EfficientNet-B7 for the Kaggle dog breed identification competition. It includes hyperparameter tuning with Optuna and training with mixed precision (AMP)."},{"cell_type":"code","execution_count":null,"id":"79622e64","metadata":{},"outputs":[],"source":"import os\nimport pandas as pd\n\n# The path variable from the previous cell stores the download location\n# Let's list its contents\ndownload_path = '/kaggle/input/competitions/dog-breed-identification'\nprint(f\"Contents of {download_path}:\")\nfor root, dirs, files in os.walk(download_path):\n    for name in files:\n        print(os.path.join(root, name))\n    for name in dirs:\n        print(os.path.join(root, name))\n\n# Read labels.csv\nlabels_df_path = os.path.join(download_path, 'labels.csv')\ntry:\n    labels_df = pd.read_csv(labels_df_path)\n    print(f\"\\nSuccessfully loaded {labels_df_path}:\")\n    print(labels_df.head())\nexcept FileNotFoundError:\n    print(f\"\\nError: {labels_df_path} not found.\")\nexcept Exception as e:\n    print(f\"\\nError loading {labels_df_path}: {e}\")\n\n# Read sample_submission.csv\nsample_sub_df_path = os.path.join(download_path, 'sample_submission.csv')\ntry:\n    sample_sub_df = pd.read_csv(sample_sub_df_path)\n    print(f\"\\nSuccessfully loaded {sample_sub_df_path}:\")\n    print(sample_sub_df.head())\nexcept FileNotFoundError:\n    print(f\"\\nError: {sample_sub_df_path} not found.\")\nexcept Exception as e:\n    print(f\"\\nError loading {sample_sub_df_path}: {e}\")"},{"cell_type":"code","execution_count":null,"id":"968849fa","metadata":{},"outputs":[],"source":"import os\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\ntrain_image_path = os.path.join(download_path, 'train')\n\n# Get a list of all image files in the train directory\nimage_files = [f for f in os.listdir(train_image_path) if f.endswith('.jpg')]\nimage_files.sort()\n\nif image_files:\n    first_image_filename = image_files[0]\n    first_image_full_path = os.path.join(train_image_path, first_image_filename)\n\n    print(f\"Opening and displaying the first image: {first_image_full_path}\")\n\n    try:\n        img = Image.open(first_image_full_path)\n        plt.imshow(img)\n        plt.title(f\"First Image from Train Folder ({first_image_filename})\")\n        plt.axis('off')\n        plt.show()\n    except Exception as e:\n        print(f\"Error opening or displaying image: {e}\")\nelse:\n    print(f\"No image files found in {train_image_path}\")"},{"cell_type":"code","execution_count":null,"id":"de405a4c","metadata":{},"outputs":[],"source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torchvision import transforms, models\nfrom torchvision.models import efficientnet_b7, EfficientNet_B7_Weights\nfrom sklearn.model_selection import train_test_split\nfrom PIL import Image\nimport os\nimport pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\nimport optuna\nimport logging\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", message=\".*can only test a child process.*\")\nwarnings.filterwarnings(\"ignore\", message=\".*Exception ignored.*\")\n\n# Check GPU availability\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\n# 1. Prepare Data\nclass DogBreedDataset(Dataset):\n    def __init__(self, df, img_dir, transform=None):\n        self.df = df\n        self.img_dir = img_dir\n        self.transform = transform\n        self.breeds = sorted(df['breed'].unique())\n        self.breed_to_idx = {breed: i for i, breed in enumerate(self.breeds)}\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_id = self.df.iloc[idx]['id']\n        img_name = f\"{img_id}.jpg\"\n        img_path = os.path.join(self.img_dir, img_name)\n        image = Image.open(img_path).convert('RGB')\n        label = self.breed_to_idx[self.df.iloc[idx]['breed']]\n        if self.transform: image = self.transform(image)\n        return image, label\n\n# EfficientNet-B7 uses 600x600 resolution\nweights = EfficientNet_B7_Weights.IMAGENET1K_V1\nbase_transforms = weights.transforms()\n\ntrain_transforms = transforms.Compose([\n    transforms.RandomResizedCrop(600),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(15),\n    transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\n# 70:30 Split\ntrain_idx, val_idx = train_test_split(\n    list(range(len(labels_df))), test_size=0.3, random_state=42, stratify=labels_df['breed']\n)\n\nfull_dataset_train = DogBreedDataset(labels_df, train_image_path, transform=train_transforms)\nfull_dataset_val = DogBreedDataset(labels_df, train_image_path, transform=base_transforms)\n\ntrain_dataset = Subset(full_dataset_train, train_idx)\nval_dataset = Subset(full_dataset_val, val_idx)\n\nNUM_WORKERS = 2\n# Single GPU configuration - B7 is larger, so use smaller batch sizes\ntrain_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, num_workers=NUM_WORKERS, pin_memory=True, persistent_workers=True)\nval_loader = DataLoader(val_dataset, batch_size=16, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True, persistent_workers=True)\n\nprint(f\"Train dataset size: {len(train_dataset)}\")\nprint(f\"Val dataset size: {len(val_dataset)}\")\nprint(f\"Number of breeds: {len(full_dataset_train.breeds)}\")"},{"cell_type":"code","execution_count":null,"id":"aabbc22e","metadata":{},"outputs":[],"source":"global_best_loss = float('inf')\ndef objective(trial):\n    global global_best_loss\n    lr = trial.suggest_float(\"lr\", 1e-5, 5e-4, log=True) \n    optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"RMSprop\"])\n    \n    print(f\"\\n[Trial {trial.number}] EfficientNet-B7 Single-GPU | lr={lr:.6f}, opt={optimizer_name} | AMP=True\")\n    \n    model = efficientnet_b7(weights=weights)\n    \n    for param in model.parameters():\n        param.requires_grad = False\n    # Unfreeze the last blocks for fine-tuning\n    for param in model.features[10:].parameters():\n        param.requires_grad = True\n    \n    in_features = model.classifier[1].in_features\n    model.classifier[1] = nn.Linear(in_features, len(full_dataset_train.breeds))\n    \n    model = model.to(device)\n    \n    optimizer = getattr(optim, optimizer_name)(filter(lambda p: p.requires_grad, model.parameters()), lr=lr)\n    criterion = nn.CrossEntropyLoss()\n\n    # Initialize GradScaler for AMP\n    scaler = torch.amp.GradScaler()\n    \n    best_trial_val_loss = float('inf')\n    patience = 4\n    counter = 0\n    \n    for epoch in range(100):\n        model.train()\n        train_loss = 0.0\n        for images, labels in tqdm(train_loader, desc=f\"Ep {epoch} [Train]\", leave=False):\n            images, labels = images.to(device), labels.to(device)\n            optimizer.zero_grad()\n    \n            # Autocast for forward pass\n            with torch.amp.autocast(device_type='cuda' if torch.cuda.is_available() else 'cpu'):\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n    \n            # Scaled backward pass and step\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n    \n            train_loss += loss.item() * images.size(0)\n    \n        model.eval()\n        val_loss, correct, total = 0.0, 0, 0\n        with torch.no_grad():\n            for images, labels in tqdm(val_loader, desc=f\"Ep {epoch} [Val]\", leave=False):\n                images, labels = images.to(device), labels.to(device)\n                with torch.amp.autocast(device_type='cuda' if torch.cuda.is_available() else 'cpu'):\n                    outputs = model(images)\n                    val_loss += criterion(outputs, labels).item() * images.size(0)\n                _, predicted = torch.max(outputs.data, 1)\n                total += labels.size(0)\n                correct += (predicted == labels).sum().item()\n    \n        epoch_val_loss = val_loss / len(val_dataset)\n        epoch_acc = 100 * correct / total\n        print(f\"Epoch {epoch}: Train Loss: {train_loss/len(train_dataset):.4f} | Val Loss: {epoch_val_loss:.4f} | Val Acc: {epoch_acc:.2f}%\")\n\n        if epoch_val_loss < best_trial_val_loss:\n            best_trial_val_loss = epoch_val_loss\n            counter = 0\n            if epoch_val_loss < global_best_loss:\n                global_best_loss = epoch_val_loss\n                # Save model state\n                torch.save(model.state_dict(), 'best_model.pth')\n        else:\n            counter += 1\n            if counter >= patience: break\n\n    return best_trial_val_loss\n\nstudy = optuna.create_study(direction=\"minimize\")\nstudy.optimize(objective, n_trials=5)"},{"cell_type":"code","execution_count":null,"id":"cb7a53e0","metadata":{},"outputs":[],"source":"if 'study' in globals() and len(study.trials) > 0:\n    best_params = study.best_params\n    print(f\"Using best hyperparameters: {best_params}\")\n\n    # 1. Setup Data: Combine train and val for final training\n    final_transform = train_transforms\n    final_dataset = DogBreedDataset(labels_df, train_image_path, transform=final_transform)\n    final_loader = DataLoader(final_dataset, batch_size=16, shuffle=True, num_workers=2, pin_memory=True)\n\n    # 2. Re-initialize Model with best architecture\n    model = efficientnet_b7(weights=EfficientNet_B7_Weights.IMAGENET1K_V1)\n    \n    # Freeze first layers, unfreeze last blocks\n    for param in model.parameters():\n        param.requires_grad = False\n    for param in model.features[10:].parameters():\n        param.requires_grad = True\n\n    in_features = model.classifier[1].in_features\n    model.classifier[1] = nn.Linear(in_features, len(final_dataset.breeds))\n    \n    model = model.to(device)\n\n    # 3. Setup Training with best params\n    optimizer = getattr(optim, best_params['optimizer'])(filter(lambda p: p.requires_grad, model.parameters()), lr=best_params['lr'])\n    criterion = nn.CrossEntropyLoss()\n    scaler = torch.cuda.amp.GradScaler()\n\n    # 4. Final Training Loop\n    print(\"Starting final training on all data...\")\n    model.train()\n    for epoch in range(20):\n        running_loss = 0.0\n        for images, labels in tqdm(final_loader, desc=f\"Final Epoch {epoch}\"):\n            images, labels = images.to(device), labels.to(device)\n            optimizer.zero_grad()\n            with torch.cuda.amp.autocast():\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            running_loss += loss.item()\n        \n        print(f\"Epoch {epoch} Loss: {running_loss/len(final_loader):.4f}\")\n\n    # 5. Save the final model\n    torch.save(model.state_dict(), 'final_best_model.pth')\n    print(\"Final model saved as 'final_best_model.pth'\")\nelse:\n    print(\"No completed Optuna study found. Please run the tuning cell first.\")"},{"cell_type":"code","execution_count":null,"id":"2d319ddb","metadata":{},"outputs":[],"source":"import torch.optim as optim\n\n# Define the hyperparameters for this single run\n# These can be adjusted based on your findings\nlr = 0.0002935680489150312 \noptimizer_name = 'RMSprop'\n\nprint(f\"\\n[Single Run] EfficientNet-B7 Single-GPU | lr={lr:.6f}, opt={optimizer_name} | AMP=True\")\n\nmodel = efficientnet_b7(weights=weights)\n\nfor param in model.parameters():\n    param.requires_grad = False\nfor param in model.features[10:].parameters():\n    param.requires_grad = True\n\nin_features = model.classifier[1].in_features\nmodel.classifier[1] = nn.Linear(in_features, len(full_dataset_train.breeds))\n\nmodel = model.to(device)\n\noptimizer = getattr(optim, optimizer_name)(filter(lambda p: p.requires_grad, model.parameters()), lr=lr)\ncriterion = nn.CrossEntropyLoss()\n\n# Initialize GradScaler for AMP\nscaler = torch.cuda.amp.GradScaler()\n\nbest_val_loss = float('inf')\npatience = 12\ncounter = 0\n\nfor epoch in range(100):\n    model.train()\n    train_loss = 0.0\n    for images, labels in tqdm(train_loader, desc=f\"Ep {epoch} [Train]\", leave=False):\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n\n        # Autocast for forward pass\n        with torch.cuda.amp.autocast():\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n\n        # Scaled backward pass and step\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        train_loss += loss.item() * images.size(0)\n\n    model.eval()\n    val_loss, correct, total = 0.0, 0, 0\n    with torch.no_grad():\n        for images, labels in tqdm(val_loader, desc=f\"Ep {epoch} [Val]\", leave=False):\n            images, labels = images.to(device), labels.to(device)\n            with torch.cuda.amp.autocast():\n                outputs = model(images)\n                val_loss += criterion(outputs, labels).item() * images.size(0)\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    epoch_val_loss = val_loss / len(val_dataset)\n    epoch_acc = 100 * correct / total\n    print(f\"Epoch {epoch}: Train Loss: {train_loss/len(train_dataset):.4f} | Val Loss: {epoch_val_loss:.4f} | Val Acc: {epoch_acc:.2f}%\")\n\n    if epoch_val_loss < best_val_loss:\n        best_val_loss = epoch_val_loss\n        counter = 0\n        # Save the model state dictionary\n        torch.save(model.state_dict(), 'best_model.pth')\n        print(f\"New best model saved with validation loss: {best_val_loss:.4f}\")\n    else:\n        counter += 1\n        if counter >= patience:\n            print(f\"Early stopping triggered after {epoch+1} epochs due to no improvement for {patience} epochs.\")\n            break\n\nprint(f\"\\nSingle run finished. Best validation loss: {best_val_loss:.4f}\")"},{"cell_type":"code","execution_count":null,"id":"a428655e","metadata":{},"outputs":[],"source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nfrom torchvision.models import efficientnet_b7, EfficientNet_B7_Weights\nfrom PIL import Image\nimport os\nimport pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\n\n# 1. Setup Device to GPU\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# 2. Re-initialize Model Architecture\nweights = EfficientNet_B7_Weights.IMAGENET1K_V1\nmodel = efficientnet_b7(weights=None)\n\n# Match the architecture used during training\nin_features = model.classifier[1].in_features\nnum_classes = 120 # Known from the dataset\nmodel.classifier[1] = nn.Linear(in_features, num_classes)\n\n# 3. Load the Saved Weights\nmodel_path = '/kaggle/working/best_model.pth'\nif os.path.exists(model_path):\n    model.load_state_dict(torch.load(model_path, map_location=device))\n    print(f\"Weights loaded from {model_path}\")\nelse:\n    print(f\"Error: {model_path} not found. Please ensure the file exists.\")\n\nmodel = model.to(device)\nmodel.eval()\n\n# 4. Define Dataset and Loader for Test Set\nclass DogBreedTestDataset(Dataset):\n    def __init__(self, img_dir, transform=None):\n        self.img_dir = img_dir\n        self.transform = transform\n        self.img_files = [f for f in os.listdir(img_dir) if f.endswith('.jpg')]\n\n    def __len__(self):\n        return len(self.img_files)\n\n    def __getitem__(self, idx):\n        img_name = self.img_files[idx]\n        img_id = img_name.split('.')[0]\n        img_path = os.path.join(self.img_dir, img_name)\n        image = Image.open(img_path).convert('RGB')\n        if self.transform: image = self.transform(image)\n        return image, img_id\n\nbase_transforms = weights.transforms()\ntest_image_path = os.path.join('/kaggle/input/competitions/dog-breed-identification/', 'test')\ntest_dataset = DogBreedTestDataset(test_image_path, transform=base_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=2)\n\n# 5. Generate Predictions\nall_predictions = []\nall_ids = []\n\nprint(\"Generating predictions for the test set...\")\nwith torch.no_grad():\n    for images, ids in tqdm(test_loader):\n        images = images.to(device)\n        outputs = model(images)\n        probs = F.softmax(outputs, dim=1)\n        all_predictions.append(probs.cpu().numpy())\n        all_ids.extend(ids)\n\n# 6. Create Submission File\npredictions_array = np.concatenate(all_predictions, axis=0)\n# Breed columns should match the order from the training 'labels_df'\nbreed_columns = sorted(pd.read_csv('/kaggle/input/competitions/dog-breed-identification/labels.csv')['breed'].unique())\n\nsubmission_df = pd.DataFrame(predictions_array, columns=breed_columns)\nsubmission_df.insert(0, 'id', all_ids)\n\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"\\nSubmission file saved as 'submission.csv'\")\ndisplay(submission_df.head())"},{"cell_type":"code","execution_count":null,"id":"9523597a","metadata":{},"outputs":[],"source":"os.getcwd()"},{"cell_type":"code","execution_count":null,"id":"f41e4dd5","metadata":{},"outputs":[],"source":"os.listdir('.')"}],"metadata":{"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":5}