{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14494031,"sourceType":"datasetVersion","datasetId":9257347},{"sourceId":4537,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3329,"modelId":986}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nos.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport warnings\nimport logging\nimport cv2\nimport json\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nfrom transformers import AutoImageProcessor, AutoModel\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n\nwarnings.filterwarnings(\"ignore\")\nlogging.getLogger(\"transformers\").setLevel(logging.ERROR)\n\n# ==========================================\n# 1. 設定與路徑搜尋\n# ==========================================\nprint(\"=\"*80)\nprint(\"🚀 啟動：Teacher-Student 知識蒸餾模型 (CSIRO Biomass)\")\nprint(\"=\"*80)\n\ndef find_path_globally(target_name, is_file=True):\n    for root, dirs, files in os.walk('/kaggle/input'):\n        if is_file:\n            if target_name in files: return os.path.join(root, target_name)\n        else:\n            if 'config.json' in files and 'dino' in root.lower(): return root\n    return None\n\n# 自動尋找路徑\nTRAIN_CSV = \"/kaggle/input/trainchange1/train_chang.csv\"\nif not os.path.exists(TRAIN_CSV):\n    TRAIN_CSV = find_path_globally(\"train_chang.csv\")\n    if not TRAIN_CSV: TRAIN_CSV = find_path_globally(\"train_standardized.csv\")\n\nTRAIN_IMG_DIR = '/kaggle/input/csiro-biomass/train'\nTEST_IMG_DIR = '/kaggle/input/csiro-biomass/test'\nTEST_CSV = find_path_globally(\"test.csv\")\nDINO_PATH = '/kaggle/input/dinov2/pytorch/giant/1'\n\n# 確保圖片路徑正確\nif not os.path.exists(TRAIN_IMG_DIR):\n    for root, dirs, files in os.walk('/kaggle/input'):\n        if 'train' in dirs and 'test' in dirs:\n            TRAIN_IMG_DIR = os.path.join(root, 'train')\n            TEST_IMG_DIR = os.path.join(root, 'test')\n            break\n\nprint(f\"📂 訓練 CSV: {TRAIN_CSV}\")\nprint(f\"📂 圖片路徑: {TRAIN_IMG_DIR}\")\nprint(f\"🤖 DINOv2: {DINO_PATH}\")\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"🔥 使用設備: {DEVICE}\")\n\n# 目標欄位 (Ground Truth)\nTARGET_COLS = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n\n# ==========================================\n# 2. 表格特徵處理器 (Tabular Processor)\n# ==========================================\nclass TabularProcessor:\n    def __init__(self):\n        self.scaler = StandardScaler()\n        self.state_ohe = OneHotEncoder(sparse_output=False, handle_unknown='ignore')\n        # 常見草種關鍵字\n        self.species_keywords = ['Clover', 'Ryegrass', 'Phalaris', 'Cocksfoot', 'Lucerne', 'Chicory', 'Tall_fescue', 'Kikuyu']\n        \n    def fit(self, df):\n        print(\"📊 正在分析表格資料分佈 (Fit)...\")\n        # 準備數值資料 (填補缺失值)\n        df_num = df[['Pre_GSHH_NDVI', 'Height_Ave_cm']].fillna(0)\n        self.scaler.fit(df_num)\n        # 準備地點資料\n        self.state_ohe.fit(df[['State']].fillna('Unknown'))\n        \n    def transform(self, df):\n        # 1. 數值特徵\n        df_num = df[['Pre_GSHH_NDVI', 'Height_Ave_cm']].fillna(0)\n        nums = self.scaler.transform(df_num)\n        \n        # 2. 時間特徵 (Date -> Sin/Cos)\n        dates = pd.to_datetime(df['Sampling_Date'])\n        months = dates.dt.month\n        # 週期性編碼\n        date_sin = np.sin(2 * np.pi * months / 12).values.reshape(-1, 1)\n        date_cos = np.cos(2 * np.pi * months / 12).values.reshape(-1, 1)\n        \n        # 3. 地點特徵\n        states = self.state_ohe.transform(df[['State']].fillna('Unknown'))\n        \n        # 4. 草種特徵 (Multi-Hot)\n        sp_list = []\n        species_series = df['Species'].fillna('').astype(str)\n        for kw in self.species_keywords:\n            has_sp = species_series.apply(lambda x: 1 if kw in x else 0).values.reshape(-1, 1)\n            sp_list.append(has_sp)\n        species_mat = np.hstack(sp_list)\n        \n        # 合併所有特徵\n        features = np.hstack([nums, date_sin, date_cos, states, species_mat])\n        return torch.tensor(features, dtype=torch.float32)\n\n# ==========================================\n# 3. 定義 Dataset\n# ==========================================\nclass DistillationDataset(Dataset):\n    def __init__(self, df, img_dir, processor, tabular_feats, target_cols):\n        self.df = df\n        self.img_dir = img_dir\n        self.processor = processor # DINO Processor\n        self.tabular_feats = tabular_feats # 預處理好的 Tensor (N, Dim)\n        self.target_cols = target_cols\n\n    def __len__(self):\n        return len(self.df)\n\n    def create_mask(self, img_array):\n        # HSV 去背\n        hsv = cv2.cvtColor(img_array, cv2.COLOR_RGB2HSV)\n        mask = cv2.inRange(hsv, np.array([5, 20, 20]), np.array([100, 255, 255]))\n        return mask\n\n    def __getitem__(self, idx):\n        try:\n            row = self.df.iloc[idx]\n            \n            # ID 處理\n            if 'sample_id_base' in row: img_id = str(row['sample_id_base']).split('__')[0]\n            else: img_id = str(row['sample_id']).split('__')[0]\n                \n            img_path = os.path.join(self.img_dir, f\"{img_id}.jpg\")\n            if not os.path.exists(img_path): img_path = img_path.replace('.jpg', '.JPG')\n            \n            if not os.path.exists(img_path): return None # 找不到圖\n\n            # 讀取圖片與去背\n            img = Image.open(img_path).convert(\"RGB\")\n            img_arr = np.array(img)\n            mask = self.create_mask(img_arr)\n            \n            processed_crops = []\n            crop_size = 500\n            for y in [0, 500]:\n                for x in [0, 500, 1000, 1500]:\n                    crop = img.crop((x, y, x+crop_size, y+crop_size))\n                    mask_crop = mask[y:y+crop_size, x:x+crop_size]\n                    \n                    # 應用 Mask\n                    c_arr = np.array(crop)\n                    m_3 = np.stack([mask_crop]*3, axis=-1)\n                    masked = np.where(m_3 > 0, c_arr, 0)\n                    processed_crops.append(Image.fromarray(masked.astype(np.uint8)))\n            \n            # DINO Preprocess\n            inputs = self.processor(images=processed_crops, return_tensors=\"pt\")\n            \n            # 獲取該行對應的表格特徵\n            tab_feat = self.tabular_feats[idx] # (Dim,)\n            \n            # 獲取 GT\n            targets = torch.tensor(row[self.target_cols].values.astype(float), dtype=torch.float32)\n            \n            return {\n                'pixel_values': inputs['pixel_values'], # (8, 3, 518, 518)\n                'tabular_feat': tab_feat,               # (Dim,)\n                'targets': targets,                     # (5,)\n                'img_id': img_id\n            }\n        except Exception as e:\n            return None\n\ndef collate_fn(batch):\n    batch = [b for b in batch if b is not None]\n    if len(batch) == 0: return None\n    return torch.utils.data.dataloader.default_collate(batch)\n\n# ==========================================\n# 4. 定義模型架構 (Teacher & Student)\n# ==========================================\n\n# 老師：看圖片特徵 + 表格特徵\nclass TeacherNet(nn.Module):\n    def __init__(self, img_dim=1536, tab_dim=20, out_dim=5):\n        super().__init__()\n        # 影像分支\n        self.img_mlp = nn.Sequential(\n            nn.Linear(img_dim, 512),\n            nn.ReLU(),\n            nn.Dropout(0.1)\n        )\n        # 表格分支\n        self.tab_mlp = nn.Sequential(\n            nn.Linear(tab_dim, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Linear(64, 64),\n            nn.ReLU()\n        )\n        # 融合分支\n        self.head = nn.Sequential(\n            nn.Linear(512 + 64, 256),\n            nn.ReLU(),\n            nn.Linear(256, 128),\n            nn.ReLU(),\n            nn.Linear(128, out_dim)\n        )\n        \n    def forward(self, x_img, x_tab):\n        h_img = self.img_mlp(x_img)\n        h_tab = self.tab_mlp(x_tab)\n        combined = torch.cat([h_img, h_tab], dim=1)\n        return self.head(combined)\n\n# 學生：只看圖片特徵\nclass StudentNet(nn.Module):\n    def __init__(self, img_dim=1536, out_dim=5):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(img_dim, 512),\n            nn.BatchNorm1d(512), # BN 幫助學生穩定訓練\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, 256),\n            nn.ReLU(),\n            nn.Linear(256, 128),\n            nn.ReLU(),\n            nn.Linear(128, out_dim)\n        )\n\n    def forward(self, x_img):\n        return self.net(x_img)\n\n# ==========================================\n# 5. 主程式流程：前處理 & 提取特徵\n# ==========================================\n\n# A. 載入與清洗資料\nprint(\"\\n[Step 1] 資料前處理...\")\ndf = pd.read_csv(TRAIN_CSV)\nfor col in TARGET_COLS:\n    df[col] = pd.to_numeric(df[col], errors='coerce')\ndf = df.dropna(subset=TARGET_COLS)\n\n# B. 表格特徵編碼\ntab_proc = TabularProcessor()\ntab_proc.fit(df)\ntrain_tab_tensor = tab_proc.transform(df) # (N_samples, Tab_Dim)\nTAB_DIM = train_tab_tensor.shape[1]\nprint(f\"✅ 表格特徵處理完成，維度: {TAB_DIM}\")\n\n# C. 載入 DINOv2\nprint(\"\\n[Step 2] 載入 DINOv2 並提取影像特徵...\")\ndino_processor = AutoImageProcessor.from_pretrained(DINO_PATH, local_files_only=True)\ndino_model = AutoModel.from_pretrained(DINO_PATH, local_files_only=True).to(DEVICE)\nif torch.cuda.device_count() > 1: dino_model = nn.DataParallel(dino_model)\ndino_model.eval()\n\n# D. 建立 Dataset & Loader\nds = DistillationDataset(df, TRAIN_IMG_DIR, dino_processor, train_tab_tensor, TARGET_COLS)\nloader = DataLoader(ds, batch_size=4, shuffle=False, num_workers=0, collate_fn=collate_fn)\n\n# E. 執行特徵提取 (一次性工作)\nall_img_feats = []\nall_tab_feats = []\nall_targets = []\n\nwith torch.no_grad():\n    for batch in tqdm(loader, desc=\"Extracting DINO Features\"):\n        if batch is None: continue\n        \n        # 1. 影像特徵提取\n        imgs = batch['pixel_values'].to(DEVICE) # (B, 8, 3, 518, 518)\n        b, n, c, h, w = imgs.shape\n        imgs = imgs.view(b*n, c, h, w)\n        \n        outputs = dino_model(pixel_values=imgs)\n        feats = outputs.last_hidden_state[:, 0, :].cpu() # (B*8, 1536)\n        feats = feats.view(b, n, -1) # (B, 8, 1536)\n        \n        # 2. 表格特徵擴展 (對齊影像切片)\n        tabs = batch['tabular_feat'] # (B, Tab_Dim)\n        gt = batch['targets']        # (B, 5)\n        \n        for i in range(b):\n            # 每一張大圖有 8 個切片，這 8 個切片共享同一組表格數據\n            curr_img_feats = feats[i] # (8, 1536)\n            curr_tab = tabs[i].repeat(n, 1) # (8, Tab_Dim)\n            curr_gt = gt[i].repeat(n, 1)    # (8, 5)\n            \n            all_img_feats.append(curr_img_feats)\n            all_tab_feats.append(curr_tab)\n            all_targets.append(curr_gt)\n\n# 合併成大 Tensor (準備訓練用)\nX_img = torch.cat(all_img_feats, dim=0) # (Total_Crops, 1536)\nX_tab = torch.cat(all_tab_feats, dim=0) # (Total_Crops, Tab_Dim)\nY_gt  = torch.cat(all_targets, dim=0)   # (Total_Crops, 5)\n\nprint(f\"✅ 特徵提取完畢: X_img={X_img.shape}, X_tab={X_tab.shape}\")\n\n# 釋放 DINO 記憶體 (後面只需訓練 MLP)\ndel dino_model\ntorch.cuda.empty_cache()\n\n# ==========================================\n# 6. 訓練階段 (Teacher -> Student) [升級版]\n# ==========================================\nimport copy\n\n# 1. 切分 訓練集 (80%) 與 驗證集 (20%)\n# 這樣我們才能知道模型是不是真的變強了，還是只是死記答案\nnum_samples = X_img.shape[0]\nindices = torch.randperm(num_samples)\nsplit = int(num_samples * 0.8) # 80% 訓練\n\ntrain_idx, val_idx = indices[:split], indices[split:]\n\n# 訓練數據\ntr_img, tr_tab, tr_gt = X_img[train_idx], X_tab[train_idx], Y_gt[train_idx]\n# 驗證數據 (考卷)\nval_img, val_tab, val_gt = X_img[val_idx], X_tab[val_idx], Y_gt[val_idx]\n\n# 建立 Loader\nbatch_size = 128\ntrain_ds = TensorDataset(tr_img, tr_tab, tr_gt)\ntrain_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True)\n\ncriterion = nn.MSELoss()\n\n# -------------------------------------------------------\n# Phase 1: 訓練 Teacher (特權模式)\n# -------------------------------------------------------\nprint(f\"\\n[Step 3] 訓練 Teacher 模型 (Epochs: 60)...\")\nteacher = TeacherNet(img_dim=1536, tab_dim=TAB_DIM).to(DEVICE)\nopt_T = torch.optim.AdamW(teacher.parameters(), lr=1e-3, weight_decay=1e-4)\nscheduler_T = torch.optim.lr_scheduler.ReduceLROnPlateau(opt_T, mode='min', factor=0.5, patience=5)\n\nbest_teacher_loss = float('inf')\nbest_teacher_weights = copy.deepcopy(teacher.state_dict())\n\nfor epoch in range(100): # 增加到 100 輪\n    teacher.train()\n    train_loss = 0\n    for bx_img, bx_tab, by in train_loader:\n        bx_img, bx_tab, by = bx_img.to(DEVICE), bx_tab.to(DEVICE), by.to(DEVICE)\n        \n        pred = teacher(bx_img, bx_tab)\n        loss = criterion(pred, by)\n        \n        opt_T.zero_grad()\n        loss.backward()\n        opt_T.step()\n        train_loss += loss.item()\n    \n    # 驗證\n    teacher.eval()\n    with torch.no_grad():\n        v_img, v_tab, v_gt = val_img.to(DEVICE), val_tab.to(DEVICE), val_gt.to(DEVICE)\n        v_pred = teacher(v_img, v_tab)\n        val_loss = criterion(v_pred, v_gt).item()\n        val_rmse = np.sqrt(val_loss)\n    \n    # 紀錄最佳模型\n    if val_loss < best_teacher_loss:\n        best_teacher_loss = val_loss\n        best_teacher_weights = copy.deepcopy(teacher.state_dict())\n    \n    scheduler_T.step(val_loss) # 學習率調整\n    \n    if (epoch+1) % 10 == 0:\n        print(f\"  Epoch {epoch+1}/60 | Train Loss: {train_loss/len(train_loader):.4f} | Val RMSE: {val_rmse:.4f}\")\n\n# 載入最佳權重並凍結\nteacher.load_state_dict(best_teacher_weights)\nteacher.eval()\nprint(f\"✅ Teacher 訓練完成 (最佳 Val RMSE: {np.sqrt(best_teacher_loss):.4f})\")\n\n\n# -------------------------------------------------------\n# Phase 2: 訓練 Student (蒸餾)\n# -------------------------------------------------------\nprint(f\"\\n[Step 4] 訓練 Student 模型 (Epochs: 100)...\")\nstudent = StudentNet(img_dim=1536).to(DEVICE)\nopt_S = torch.optim.AdamW(student.parameters(), lr=1e-3, weight_decay=1e-4)\nscheduler_S = torch.optim.lr_scheduler.ReduceLROnPlateau(opt_S, mode='min', factor=0.5, patience=8)\n\nALPHA = 0.7 \nEPOCHS_S = 150 # 增加到 100 輪\n\nbest_student_loss = float('inf')\nbest_student_weights = copy.deepcopy(student.state_dict())\n\nfor epoch in range(EPOCHS_S):\n    student.train()\n    train_loss = 0\n    \n    for bx_img, bx_tab, by in train_loader:\n        bx_img, bx_tab, by = bx_img.to(DEVICE), bx_tab.to(DEVICE), by.to(DEVICE)\n        \n        # 1. 學生預測\n        pred_student = student(bx_img)\n        \n        # 2. 老師預測\n        with torch.no_grad():\n            pred_teacher = teacher(bx_img, bx_tab)\n            \n        # 3. 混合 Loss\n        loss_gt = criterion(pred_student, by)\n        loss_distill = criterion(pred_student, pred_teacher)\n        loss = (ALPHA * loss_gt) + ((1 - ALPHA) * loss_distill)\n        \n        opt_S.zero_grad()\n        loss.backward()\n        opt_S.step()\n        train_loss += loss.item()\n        \n    # 驗證 (只看學生對不對)\n    student.eval()\n    with torch.no_grad():\n        v_img, v_gt = val_img.to(DEVICE), val_gt.to(DEVICE)\n        v_pred = student(v_img)\n        val_loss = criterion(v_pred, v_gt).item()\n        val_rmse = np.sqrt(val_loss)\n\n    if val_loss < best_student_loss:\n        best_student_loss = val_loss\n        best_student_weights = copy.deepcopy(student.state_dict())\n        \n    scheduler_S.step(val_loss)\n\n    if (epoch+1) % 10 == 0:\n        print(f\"  Epoch {epoch+1}/{EPOCHS_S} | Train Loss: {train_loss/len(train_loader):.4f} | Val RMSE: {val_rmse:.4f}\")\n\n# 載入最佳學生模型\nstudent.load_state_dict(best_student_weights)\nstudent.eval()\nprint(f\"✅ Student 訓練完成 (最佳 Val RMSE: {np.sqrt(best_student_loss):.4f})\")\n\n# ==========================================\n# 7. 預測測試集 (只用 Student)\n# ==========================================\nprint(\"\\n[Step 5] 預測測試集 (Deployment)...\")\n\n# 這裡需要重新載入 DINO 因為測試集要跑圖\ndino_model = AutoModel.from_pretrained(DINO_PATH, local_files_only=True).to(DEVICE)\nif torch.cuda.device_count() > 1: dino_model = nn.DataParallel(dino_model)\ndino_model.eval()\n\n# 測試集 Dataset (不需要表格資料，只要圖)\nclass TestDataset(Dataset):\n    def __init__(self, ids, img_dir, processor):\n        self.ids = ids\n        self.img_dir = img_dir\n        self.processor = processor\n    \n    def __len__(self): return len(self.ids)\n    \n    def __getitem__(self, idx):\n        try:\n            img_id = self.ids[idx]\n            path = os.path.join(self.img_dir, f\"{img_id}.jpg\")\n            if not os.path.exists(path): path = path.replace('.jpg', '.JPG')\n            \n            img = Image.open(path).convert(\"RGB\")\n            # 簡單 HSV Mask (測試時也要做)\n            arr = np.array(img)\n            hsv = cv2.cvtColor(arr, cv2.COLOR_RGB2HSV)\n            mask = cv2.inRange(hsv, np.array([5,20,20]), np.array([100,255,255]))\n            \n            crops = []\n            for y in [0, 500]:\n                for x in [0, 500, 1000, 1500]:\n                    c = img.crop((x, y, x+500, y+500))\n                    mc = mask[y:y+500, x:x+500]\n                    c_arr = np.array(c)\n                    m3 = np.stack([mc]*3, axis=-1)\n                    masked = np.where(m3>0, c_arr, 0)\n                    crops.append(Image.fromarray(masked.astype(np.uint8)))\n            \n            inputs = self.processor(images=crops, return_tensors=\"pt\")\n            return {'pixel_values': inputs['pixel_values'], 'img_id': img_id}\n        except: return None\n\ntest_df = pd.read_csv(TEST_CSV)\nunique_test_ids = test_df['sample_id'].apply(lambda x: x.split('__')[0]).unique()\ntest_ds = TestDataset(unique_test_ids, TEST_IMG_DIR, dino_processor)\ntest_loader = DataLoader(test_ds, batch_size=4, shuffle=False, num_workers=0, collate_fn=collate_fn)\n\npredictions = {}\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader, desc=\"Predicting Test Set\"):\n        if batch is None: continue\n        \n        imgs = batch['pixel_values'].to(DEVICE) # (B, 8, 3, 518, 518)\n        ids = batch['img_id']\n        b, n, c, h, w = imgs.shape\n        imgs = imgs.view(b*n, c, h, w)\n        \n        # 1. DINO 特徵\n        out = dino_model(pixel_values=imgs)\n        feats = out.last_hidden_state[:, 0, :] # (B*8, 1536)\n        \n        # 2. Student 預測 (只看 DINO 特徵)\n        preds = student(feats) # (B*8, 5)\n        preds = preds.view(b, n, -1) # (B, 8, 5)\n        \n        # 3. 平均 8 張切片\n        avg_preds = torch.mean(preds, dim=1).cpu().numpy() # (B, 5)\n        \n        for i, img_id in enumerate(ids):\n            # 確保不為負值\n            p = np.clip(avg_preds[i], 0, None)\n            predictions[img_id] = p\n\n# 生成 Submission\nsubmission_data = []\nfor _, row in test_df.iterrows():\n    sid = row['sample_id']\n    img_id, t_name = sid.split('__')\n    val = predictions.get(img_id, [0.0]*5)[TARGET_COLS.index(t_name)]\n    submission_data.append({'sample_id': sid, 'target': val})\n\npd.DataFrame(submission_data).to_csv('submission.csv', index=False)\nprint(\"\\n✅ 完成！Knowledge Distillation Submission 已生成！\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-14T10:52:44.609744Z","iopub.execute_input":"2026-01-14T10:52:44.610056Z","iopub.status.idle":"2026-01-14T10:59:01.660925Z","shell.execute_reply.started":"2026-01-14T10:52:44.610028Z","shell.execute_reply":"2026-01-14T10:59:01.660131Z"}},"outputs":[],"execution_count":null}]}