{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":4537,"sourceType":"modelInstanceVersion","modelInstanceId":3329,"modelId":986}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-06T12:27:01.910788Z","iopub.execute_input":"2026-01-06T12:27:01.911078Z","iopub.status.idle":"2026-01-06T12:27:04.278170Z","shell.execute_reply.started":"2026-01-06T12:27:01.911046Z","shell.execute_reply":"2026-01-06T12:27:04.277543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!pip install transformers==4.57.1\n\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm import tqdm\nimport torchvision\nimport os\nfrom torchvision.datasets import ImageFolder\nfrom torch.utils.data import DataLoader, Subset, Dataset\nimport torchvision.transforms as transforms\nfrom PIL import Image\n\ntry:\n    from transformers import AutoModel, AutoConfig\nexcept ImportError:\n    print(\"Error: Transformers library not found. Please install it or check your environment.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T12:27:04.279856Z","iopub.execute_input":"2026-01-06T12:27:04.280198Z","iopub.status.idle":"2026-01-06T12:27:22.269200Z","shell.execute_reply.started":"2026-01-06T12:27:04.280173Z","shell.execute_reply":"2026-01-06T12:27:22.268563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\n# --- CONFIGURATION ---\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nCACHE_DIR = \"/kaggle/working/features\"\nIMAGE_DIR = \"/kaggle/input/csiro-biomass/train\"\nTEST_IMAGE_DIR = \"/kaggle/input/csiro-biomass/test\"\nMODEL_PATH = \"/kaggle/input/dinov2/pytorch/giant/1\"\n\n# Create cache directory if it doesn't exist\nos.makedirs(CACHE_DIR, exist_ok=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T12:27:22.270069Z","iopub.execute_input":"2026-01-06T12:27:22.270601Z","iopub.status.idle":"2026-01-06T12:27:22.355787Z","shell.execute_reply.started":"2026-01-06T12:27:22.270563Z","shell.execute_reply":"2026-01-06T12:27:22.354979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Loading model from {MODEL_PATH}...\")\n\n# Load configuration first to ensure it's valid\ntry:\n    config = AutoConfig.from_pretrained(MODEL_PATH)\n    backbone = AutoModel.from_pretrained(MODEL_PATH, config=config)\nexcept OSError:\n    print(\"Could not load local model. Trying internet download (will fail if offline)...\")\n    backbone = AutoModel.from_pretrained(\"facebookresearch/dinov2-giant\")\nexcept NameError:\n    # If AutoModel fails, fallback to torch hub (alternative method)\n    print(\"Transformers failed. Trying Torch Hub...\")\n    backbone = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitg14')\n    \nbackbone.to(DEVICE)\nbackbone.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T12:27:22.356902Z","iopub.execute_input":"2026-01-06T12:27:22.357221Z","iopub.status.idle":"2026-01-06T12:28:12.932692Z","shell.execute_reply.started":"2026-01-06T12:27:22.357196Z","shell.execute_reply":"2026-01-06T12:28:12.932029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocess = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T12:28:12.933502Z","iopub.execute_input":"2026-01-06T12:28:12.934096Z","iopub.status.idle":"2026-01-06T12:28:12.938796Z","shell.execute_reply.started":"2026-01-06T12:28:12.934070Z","shell.execute_reply":"2026-01-06T12:28:12.938011Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ImageOnlyDataset(Dataset):\n    def __init__(self, image_dir, transform=None):\n        self.image_ids = [f.split('.')[0] for f in os.listdir(image_dir) if f.endswith('.jpg')]\n        self.image_dir = image_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        img_id = self.image_ids[idx]\n        img_path = os.path.join(self.image_dir, f\"{img_id}.jpg\")\n        image = Image.open(img_path).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n        return image, img_id\n\n# --- 3. EXTRACTION LOOP ---\ndef extract_and_cache(directory, desc=\"Extracting\"):\n    # Filter out images that are already cached\n    # We check if ID123.pt exists in CACHE_DIR\n    all_files = [f for f in os.listdir(directory) if f.endswith('.jpg')]\n    to_process = [f for f in all_files if not os.path.exists(f\"{CACHE_DIR}/{f.split('.')[0]}.pt\")]\n    \n    if len(to_process) == 0:\n        print(f\"All images in {desc} are already cached!\")\n        return\n\n    # Create a temporary dataset for just the missing files\n    # (Simplified for this snippet: We just loop the subset)\n    \n    print(f\"Processing {len(to_process)} new images for {desc}...\")\n    \n    # Process in batches for speed\n    batch_size = 16\n    \n    # Helper to chunk the list\n    chunks = [to_process[i:i + batch_size] for i in range(0, len(to_process), batch_size)]\n    \n    with torch.no_grad():\n        for chunk in tqdm(chunks, desc=desc):\n            images = []\n            ids = []\n            \n            # Load batch\n            for filename in chunk:\n                img_id = filename.split('.')[0]\n                img_path = os.path.join(directory, filename)\n                img = Image.open(img_path).convert('RGB')\n                images.append(preprocess(img))\n                ids.append(img_id)\n            \n            # Stack and Move to GPU\n            batch_tensor = torch.stack(images).to(DEVICE)\n            \n            # Forward Pass\n            outputs = backbone(batch_tensor)\n            \n            # Get Embeddings\n            # method A: CLS Token (outputs.last_hidden_state[:, 0, :])\n            # method B: Average Pooling (Better for Biomass)\n            embeddings = outputs.last_hidden_state.mean(dim=1) # Shape: (Batch, 1536)\n            \n            # Save to Disk\n            for i, img_id in enumerate(ids):\n                # Save as a CPU tensor to save GPU memory\n                torch.save(embeddings[i].cpu(), f\"{CACHE_DIR}/{img_id}.pt\")\n\n# Run Extraction for Train and Test\nextract_and_cache(IMAGE_DIR, desc=\"Train Images\")\nextract_and_cache(TEST_IMAGE_DIR, desc=\"Test Images\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T12:28:12.939606Z","iopub.execute_input":"2026-01-06T12:28:12.939866Z","iopub.status.idle":"2026-01-06T12:29:37.765089Z","shell.execute_reply.started":"2026-01-06T12:28:12.939838Z","shell.execute_reply":"2026-01-06T12:29:37.764400Z"}},"outputs":[],"execution_count":null}]}