It seems that a big shake occurred as a result.The shake may have been expected, as there were many well-known Kagglers with fewer subs in the discussions and LB.\nDue to personal reasons, I was only able to participate for two weeks, but the problem setting was simple, and the relationships between the subjects allowed for a variety of approaches, so it was a fun competition, and I would like to express my gratitude to the organizers.
\nclass AttentionPool1d(nn.Module):\n def __init__(self, in_features, hidden_dim=None):\n super().__init__()\n h_dim = hidden_dim or (in_features // 2)\n\n self.att_net = nn.Sequential(\n nn.Conv1d(in_features, h_dim, kernel_size=1),\n nn.Tanh(),\n nn.Conv1d(h_dim, 1, kernel_size=1)\n )\n\n def forward(self, x):\n scores = self.att_net(x) \n weights = F.softmax(scores, dim=-1) \n return (x * weights).sum(dim=-1)\n\nclass BiomassModelv2(nn.Module):\n def __init__(self, model_name, pretrained=True, backbone_path=None):\n super().__init__()\n self.model_name = model_name\n self.backbone_path = backbone_path\n\n # 1. Load Backbone\n self.backbone = timm.create_model(self.model_name, pretrained=False, num_classes=0, global_pool='')\n\n if hasattr(self.backbone, 'set_grad_checkpointing'):\n self.backbone.set_grad_checkpointing(True)\n print(\"✓ Gradient Checkpointing enabled\")\n\n nf = self.backbone.num_features\n\n # 2. Mamba Fusion\n self.fusion = nn.Sequential(\n LocalMambaBlock(nf, kernel_size=5, dropout=0.1),\n LocalMambaBlock(nf, kernel_size=5, dropout=0.1)\n )\n\n # 3. Pooling (Attention Pooling)\n self.pool = AttentionPool1d(in_features=nf)\n\n # 4. Heads\n self.head_green_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n self.head_clover_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n self.head_dead_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n\n if pretrained:\n self.load_pretrained()\n\n def load_pretrained(self):\n try:\n if self.backbone_path and os.path.exists(self.backbone_path):\n print(f\"Loading backbone weights from local file: {self.backbone_path}\")\n sd = torch.load(self.backbone_path, map_location='cpu')\n if 'model' in sd: sd = sd['model']\n elif 'state_dict' in sd: sd = sd['state_dict']\n else:\n print(\"Downloading backbone weights...\")\n sd = timm.create_model(self.model_name, pretrained=True, num_classes=0, global_pool='').state_dict()\n\n self.backbone.load_state_dict(sd, strict=False)\n print('Pretrained weights loaded.')\n except Exception as e:\n print(f'Warning: pretrained load failed: {e}')\n\n def forward(self, left, right):\n # 1. Extract & Concat\n x_l = self.backbone(left)\n x_r = self.backbone(right)\n x_cat = torch.cat([x_l, x_r], dim=1) # (B, 2N, D)\n\n # 2. Fusion\n x_fused = self.fusion(x_cat) # (B, 2N, D)\n\n # 3. Attention Pooling\n x_pool = self.pool(x_fused.transpose(1, 2))\n\n # 4. Prediction Heads\n green = self.head_green_raw(x_pool)\n clover = self.head_clover_raw(x_pool)\n dead = self.head_dead_raw(x_pool)\n\n # Summation logic\n gdm = green + clover\n total = gdm + dead\n\n return total, gdm, green, clover, dead\n| Fold | \nGlobal R² | \nAvg R² | \n
|---|---|---|
| Fold 0 | \n0.7440 | \n0.6160 | \n
| Fold 1 | \n0.7007 | \n0.4397 | \n
| Fold 2 | \n0.8296 | \n0.7280 | \n
| Fold 3 | \n0.7196 | \n0.5279 | \n
| Fold 4 | \n0.8055 | \n0.6567 | \n
def get_train_transforms():\n return A.Compose([\n A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n A.HorizontalFlip(p=0.5),\n A.VerticalFlip(p=0.5),\n A.RandomRotate90(p=0.5),\n A.Rotate(limit=(-10, 10), p=0.3, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101),\n A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.05, p=0.5),\n A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n A.RandomBrightnessContrast(brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2), p=0.3),\n A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.3, rotate_limit=15, border_mode=0, p=0.50),\n ToTensorV2()\n ], p=1.0)\n(e.g.)\nfrom timm.utils import ModelEmaV2\nema = ModelEmaV2(base_model, decay=0.999)\nclass FiLMV2(nn.Module):\n def __init__(self, feat_dim):\n super().__init__()\n self.mlp = nn.Sequential(\n nn.Linear(feat_dim, feat_dim), \n nn.ReLU(inplace=True), \n nn.Linear(feat_dim, feat_dim * 2)\n )\n\n def forward(self, context):\n gamma_beta = self.mlp(context)\n return torch.chunk(gamma_beta, 2, dim=1)\nclass MultiTargetRegressorV2(nn.Module):\n def __init__(self, model_name, pretrained=True, num_classes=3, dropout=0.2, freeze_backbone=False):\n super().__init__()\n self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='avg')\n self.feat_dim = self.backbone.num_features \n self.film = FiLMV2(self.feat_dim)\n self.dropout = nn.Dropout(dropout)\n\n # Left + Right + Diff + Prod\n fusion_dim = self.feat_dim * 4\n\n def make_head():\n return nn.Sequential(\n nn.Linear(fusion_dim, 512),\n nn.BatchNorm1d(512),\n nn.ReLU(inplace=True),\n nn.Dropout(dropout),\n nn.Linear(512, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, 1),\n )\n\n self.head_green = make_head()\n self.head_clover = make_head()\n self.head_dead = make_head()\n\n self.softplus = nn.Softplus(beta=1.0)\n\n if freeze_backbone:\n for p in self.backbone.parameters():\n p.requires_grad = False\n\n def forward(self, left_img, right_img):\n left_feat = self.backbone(left_img)\n right_feat = self.backbone(right_img)\n\n context = (left_feat + right_feat) / 2\n gamma, beta = self.film(context)\n\n left_feat_mod = left_feat * (1 + gamma) + beta\n right_feat_mod = right_feat * (1 + gamma) + beta\n\n diff_feat = torch.abs(left_feat_mod - right_feat_mod)\n prod_feat = left_feat_mod * right_feat_mod\n\n combined = torch.cat([left_feat_mod, right_feat_mod, diff_feat, prod_feat], dim=1)\n\n green = self.softplus(self.head_green(combined)) \n clover = self.softplus(self.head_clover(combined))\n dead = self.softplus(self.head_dead(combined)) \n\n logits = torch.cat([green, clover, dead], dim=1)\n\n return logits\n transform = transforms.Compose([\n transforms.Resize(target_image_size),\n transforms.RandomHorizontalFlip(p=0.5),\n transforms.RandomVerticalFlip(p=0.5),\n transforms.RandomApply([transforms.ColorJitter(\n brightness=0.1, \n contrast=0.1, \n saturation=0.1, \n hue=0.05)], p=0.5),\n\n transforms.RandomApply([transforms.RandomRotation(\n degrees=10, \n interpolation=transforms.InterpolationMode.BILINEAR\n )], p=0.3),\n\n transforms.RandomApply([transforms.RandomAffine(\n degrees=15,\n translate=(0.1, 0.1),\n scale=(0.7, 1.3),\n interpolation=transforms.InterpolationMode.BILINEAR\n )], p=0.5),\n transforms.RandomApply([transforms.RandomChoice([\n transforms.GaussianBlur(kernel_size=5),\n transforms.ElasticTransform(alpha=50.0),\n transforms.RandomPerspective(distortion_scale=0.5, p=1.0) \n ])], p=0.3),\n\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n| Fold | \nweighted_r2 | \n
|---|---|
| Fold1 | \n0.7996 | \n
| Fold2 | \n0.7600 | \n
| Fold3 | \n0.7161 | \n
| Fold4 | \n0.6141 | \n
| Fold5 | \n0.7435 | \n
(e.g.)\nsubmission.loc[submission['sample_id'].str.endswith('Dry_Clover_g'),'target']*=0.8\nsubmission.to_csv(OUTPUT_FILE, index=False)\nIt seems that a big shake occurred as a result.The shake may have been expected, as there were many well-known Kagglers with fewer subs in the discussions and LB.\nDue to personal reasons, I was only able to participate for two weeks, but the problem setting was simple, and the relationships between the subjects allowed for a variety of approaches, so it was a fun competition, and I would like to express my gratitude to the organizers.
\nclass AttentionPool1d(nn.Module):\n def __init__(self, in_features, hidden_dim=None):\n super().__init__()\n h_dim = hidden_dim or (in_features // 2)\n\n self.att_net = nn.Sequential(\n nn.Conv1d(in_features, h_dim, kernel_size=1),\n nn.Tanh(),\n nn.Conv1d(h_dim, 1, kernel_size=1)\n )\n\n def forward(self, x):\n scores = self.att_net(x) \n weights = F.softmax(scores, dim=-1) \n return (x * weights).sum(dim=-1)\n\nclass BiomassModelv2(nn.Module):\n def __init__(self, model_name, pretrained=True, backbone_path=None):\n super().__init__()\n self.model_name = model_name\n self.backbone_path = backbone_path\n\n # 1. Load Backbone\n self.backbone = timm.create_model(self.model_name, pretrained=False, num_classes=0, global_pool='')\n\n if hasattr(self.backbone, 'set_grad_checkpointing'):\n self.backbone.set_grad_checkpointing(True)\n print(\"✓ Gradient Checkpointing enabled\")\n\n nf = self.backbone.num_features\n\n # 2. Mamba Fusion\n self.fusion = nn.Sequential(\n LocalMambaBlock(nf, kernel_size=5, dropout=0.1),\n LocalMambaBlock(nf, kernel_size=5, dropout=0.1)\n )\n\n # 3. Pooling (Attention Pooling)\n self.pool = AttentionPool1d(in_features=nf)\n\n # 4. Heads\n self.head_green_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n self.head_clover_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n self.head_dead_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n\n if pretrained:\n self.load_pretrained()\n\n def load_pretrained(self):\n try:\n if self.backbone_path and os.path.exists(self.backbone_path):\n print(f\"Loading backbone weights from local file: {self.backbone_path}\")\n sd = torch.load(self.backbone_path, map_location='cpu')\n if 'model' in sd: sd = sd['model']\n elif 'state_dict' in sd: sd = sd['state_dict']\n else:\n print(\"Downloading backbone weights...\")\n sd = timm.create_model(self.model_name, pretrained=True, num_classes=0, global_pool='').state_dict()\n\n self.backbone.load_state_dict(sd, strict=False)\n print('Pretrained weights loaded.')\n except Exception as e:\n print(f'Warning: pretrained load failed: {e}')\n\n def forward(self, left, right):\n # 1. Extract & Concat\n x_l = self.backbone(left)\n x_r = self.backbone(right)\n x_cat = torch.cat([x_l, x_r], dim=1) # (B, 2N, D)\n\n # 2. Fusion\n x_fused = self.fusion(x_cat) # (B, 2N, D)\n\n # 3. Attention Pooling\n x_pool = self.pool(x_fused.transpose(1, 2))\n\n # 4. Prediction Heads\n green = self.head_green_raw(x_pool)\n clover = self.head_clover_raw(x_pool)\n dead = self.head_dead_raw(x_pool)\n\n # Summation logic\n gdm = green + clover\n total = gdm + dead\n\n return total, gdm, green, clover, dead\n| Fold | \nGlobal R² | \nAvg R² | \n
|---|---|---|
| Fold 0 | \n0.7440 | \n0.6160 | \n
| Fold 1 | \n0.7007 | \n0.4397 | \n
| Fold 2 | \n0.8296 | \n0.7280 | \n
| Fold 3 | \n0.7196 | \n0.5279 | \n
| Fold 4 | \n0.8055 | \n0.6567 | \n
def get_train_transforms():\n return A.Compose([\n A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n A.HorizontalFlip(p=0.5),\n A.VerticalFlip(p=0.5),\n A.RandomRotate90(p=0.5),\n A.Rotate(limit=(-10, 10), p=0.3, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101),\n A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.05, p=0.5),\n A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n A.RandomBrightnessContrast(brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2), p=0.3),\n A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.3, rotate_limit=15, border_mode=0, p=0.50),\n ToTensorV2()\n ], p=1.0)\n(e.g.)\nfrom timm.utils import ModelEmaV2\nema = ModelEmaV2(base_model, decay=0.999)\nclass FiLMV2(nn.Module):\n def __init__(self, feat_dim):\n super().__init__()\n self.mlp = nn.Sequential(\n nn.Linear(feat_dim, feat_dim), \n nn.ReLU(inplace=True), \n nn.Linear(feat_dim, feat_dim * 2)\n )\n\n def forward(self, context):\n gamma_beta = self.mlp(context)\n return torch.chunk(gamma_beta, 2, dim=1)\nclass MultiTargetRegressorV2(nn.Module):\n def __init__(self, model_name, pretrained=True, num_classes=3, dropout=0.2, freeze_backbone=False):\n super().__init__()\n self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='avg')\n self.feat_dim = self.backbone.num_features \n self.film = FiLMV2(self.feat_dim)\n self.dropout = nn.Dropout(dropout)\n\n # Left + Right + Diff + Prod\n fusion_dim = self.feat_dim * 4\n\n def make_head():\n return nn.Sequential(\n nn.Linear(fusion_dim, 512),\n nn.BatchNorm1d(512),\n nn.ReLU(inplace=True),\n nn.Dropout(dropout),\n nn.Linear(512, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, 1),\n )\n\n self.head_green = make_head()\n self.head_clover = make_head()\n self.head_dead = make_head()\n\n self.softplus = nn.Softplus(beta=1.0)\n\n if freeze_backbone:\n for p in self.backbone.parameters():\n p.requires_grad = False\n\n def forward(self, left_img, right_img):\n left_feat = self.backbone(left_img)\n right_feat = self.backbone(right_img)\n\n context = (left_feat + right_feat) / 2\n gamma, beta = self.film(context)\n\n left_feat_mod = left_feat * (1 + gamma) + beta\n right_feat_mod = right_feat * (1 + gamma) + beta\n\n diff_feat = torch.abs(left_feat_mod - right_feat_mod)\n prod_feat = left_feat_mod * right_feat_mod\n\n combined = torch.cat([left_feat_mod, right_feat_mod, diff_feat, prod_feat], dim=1)\n\n green = self.softplus(self.head_green(combined)) \n clover = self.softplus(self.head_clover(combined))\n dead = self.softplus(self.head_dead(combined)) \n\n logits = torch.cat([green, clover, dead], dim=1)\n\n return logits\n transform = transforms.Compose([\n transforms.Resize(target_image_size),\n transforms.RandomHorizontalFlip(p=0.5),\n transforms.RandomVerticalFlip(p=0.5),\n transforms.RandomApply([transforms.ColorJitter(\n brightness=0.1, \n contrast=0.1, \n saturation=0.1, \n hue=0.05)], p=0.5),\n\n transforms.RandomApply([transforms.RandomRotation(\n degrees=10, \n interpolation=transforms.InterpolationMode.BILINEAR\n )], p=0.3),\n\n transforms.RandomApply([transforms.RandomAffine(\n degrees=15,\n translate=(0.1, 0.1),\n scale=(0.7, 1.3),\n interpolation=transforms.InterpolationMode.BILINEAR\n )], p=0.5),\n transforms.RandomApply([transforms.RandomChoice([\n transforms.GaussianBlur(kernel_size=5),\n transforms.ElasticTransform(alpha=50.0),\n transforms.RandomPerspective(distortion_scale=0.5, p=1.0) \n ])], p=0.3),\n\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n| Fold | \nweighted_r2 | \n
|---|---|
| Fold1 | \n0.7996 | \n
| Fold2 | \n0.7600 | \n
| Fold3 | \n0.7161 | \n
| Fold4 | \n0.6141 | \n
| Fold5 | \n0.7435 | \n
(e.g.)\nsubmission.loc[submission['sample_id'].str.endswith('Dry_Clover_g'),'target']*=0.8\nsubmission.to_csv(OUTPUT_FILE, index=False)\nAmazing😶I think what you did is quite reasonable. Why did you lose so much? Was it because the final submitted model was too small to pull back the 0.8 impact of yours?
", "rawMarkdown": "Amazing😶I think what you did is quite reasonable. Why did you lose so much? Was it because the final submitted model was too small to pull back the 0.8 impact of yours?", "votes": 1, "replies": [ { "id": 3398533, "postDate": "2026-01-29T09:28:16.023Z", "content": "Hi! Thank you for your comment.
\nThe issue I was unable to solve was that I was unable to implement an appropriate validation strategy.\nThe division was performed using StratifiedGroupKFold, but as shown in the table above, the Fold1 Avg R² was very low at 0.4397, and the variance between folds was also large.
\nI am aware of this issue, and have considered the following solutions:
\nUnfortunately, I didn't have the time to work on these experiments, which I think is one of the reasons why I failed.
", "rawMarkdown": "Hi! Thank you for your comment.\n\nThe issue I was unable to solve was that I was unable to implement an appropriate **validation strategy**.\nThe division was performed using StratifiedGroupKFold, but as shown in the table above, the **Fold1 Avg R² was very low at 0.4397**, and the variance between folds was also large.\n\nI am aware of this issue, and have considered the following solutions: \n* (1) Use a lightweight model and try my best to find an appropriate seed. \n* (2) Multi-layer the split for more appropriate division. \n* (3) Discard folds with too low CV during inference.\n\nUnfortunately, I didn't have the time to work on these experiments, which I think is one of the reasons why I failed.", "votes": 1, "replies": [ { "id": 3398549, "postDate": "2026-01-29T10:01:30.457Z", "rawMarkdown": "", "isDeleted": true }, { "id": 3398550, "postDate": "2026-01-29T10:02:10.380Z", "content": "StratifiedGroupKFold(random_state=82947501)、groups = df_wide['Sampling_Date']、y_stratify = df_wide['State'] —— didn't these work for you? They seemed to have a slight effect in my experiments：
\nFold 0: Global R² = 0.7916, Avg R² = 0.7155
\nFold 1: Global R² = 0.7782, Avg R² = 0.6266
\nFold 2: Global R² = 0.8941, Avg R² = 0.8206
\nFold 3: Global R² = 0.8060, Avg R² = 0.6990
\nFold 4: Global R² = 0.8405, Avg R² = 0.6748
\nI also think discarding a fold is a risky move — even if it's a reluctant choice. If all the models you trained end up discarding the same fold, you’ll definitely lose some valuable information. That’s why I opted for a slight weighting operation instead.
", "rawMarkdown": "StratifiedGroupKFold(random_state=82947501)、groups = df_wide['Sampling_Date']、y_stratify = df_wide['State'] —— didn't these work for you? They seemed to have a slight effect in my experiments：\n\nFold 0: Global R² = 0.7916, Avg R² = 0.7155 \n\nFold 1: Global R² = 0.7782, Avg R² = 0.6266\n\n Fold 2: Global R² = 0.8941, Avg R² = 0.8206\n\n Fold 3: Global R² = 0.8060, Avg R² = 0.6990 \n\nFold 4: Global R² = 0.8405, Avg R² = 0.6748\n\n I also think discarding a fold is a risky move — even if it's a reluctant choice. If all the models you trained end up discarding the same fold, you’ll definitely lose some valuable information. That’s why I opted for a slight weighting operation instead.", "votes": 1 }, { "id": 3398559, "postDate": "2026-01-29T10:19:20.040Z", "content": "Wow, that's a pretty good result!\nYes, I'm also using the Split in the public notebook you mentioned(SGKF).\nHowever, I still think that the difference in average R² score between folds of around 0.2 is a significant issue.
\nYou're right. I don't like the Fold Selection approach either, as it increases the risk of overfitting to LB.
", "rawMarkdown": "Wow, that's a pretty good result!\nYes, I'm also using the Split in the public notebook you mentioned(SGKF).\nHowever, I still think that the difference in average R² score between folds of around 0.2 is a significant issue.\n\nYou're right. I don't like the Fold Selection approach either, as it increases the risk of overfitting to LB.", "votes": 1 }, { "id": 3398593, "postDate": "2026-01-29T11:03:34.773Z", "content": "In this competition, I use a very simple validation strategy: stratified by State and choosing a seed so that the mean and standard deviation of the labels are similar across all 5 folds. There are a few main reasons: first, the dataset is too small, so if we split it in an extreme way, such as grouping by date or by state, the label distribution would become heavily distorted. Second, the host has confirmed that there are no new species in the public and private test sets. Some states have a certain species while others don’t. Grouping by date also risks missing some species or not having all four seasons represented in the training set. As I understand it, each date group corresponds to a trip to a particular farm to collect samples, and it’s possible that some clover species exist only in certain farms.
\nBasically, my split is almost certainly suffering from data leakage, and I accept that. To mitigate this, I use the CV–LB correlation and the LB score to evaluate the model.
", "rawMarkdown": "In this competition, I use a very simple validation strategy: stratified by State and choosing a seed so that the mean and standard deviation of the labels are similar across all 5 folds. There are a few main reasons: first, the dataset is too small, so if we split it in an extreme way, such as grouping by date or by state, the label distribution would become heavily distorted. Second, the host has confirmed that there are no new species in the public and private test sets. Some states have a certain species while others don’t. Grouping by date also risks missing some species or not having all four seasons represented in the training set. As I understand it, each date group corresponds to a trip to a particular farm to collect samples, and it’s possible that some clover species exist only in certain farms.\n\nBasically, my split is almost certainly suffering from data leakage, and I accept that. To mitigate this, I use the CV–LB correlation and the LB score to evaluate the model.", "votes": 2 }, { "id": 3398620, "postDate": "2026-01-29T12:21:39.963Z", "content": "I also want to ask you how you designed the network architecture. It seems that this is quite uncommon. Regarding this \"diff\", was this idea derived from a paper, or was it just a sudden inspiration? I feel that I don't have much inspiration when it comes to design.
", "rawMarkdown": "I also want to ask you how you designed the network architecture. It seems that this is quite uncommon. Regarding this \"diff\", was this idea derived from a paper, or was it just a sudden inspiration? I feel that I don't have much inspiration when it comes to design.", "votes": 1 }, { "id": 3398623, "postDate": "2026-01-29T12:26:41.397Z", "content": "Thank you for your advice. And congratulations on getting second place, that's amazing!\nAs a \"seed search\" approach, searching to minimize variance between folds seems important for this dataset.
", "rawMarkdown": "Thank you for your advice. And congratulations on getting second place, that's amazing!\nAs a \"seed search\" approach, searching to minimize variance between folds seems important for this dataset.", "votes": 1 }, { "id": 3398629, "postDate": "2026-01-29T12:44:27.817Z", "content": "The ideas for the model definition are based on excellent solutions and explanations from participants in past competitions.
\nFor example, GeM is often used as a pooling trend, so I implemented and experimented with it in this competition. (However, the results were not good, so I did not adopt it.)
\nclass GeM(nn.Module):\n def __init__(self, p=3, eps=1e-6):\n super(GeM, self).__init__()\n self.p = nn.Parameter(torch.ones(1) * p)\n self.eps = eps\n\n def forward(self, x):\n x = torch.clamp(x, min=self.eps)\n return F.avg_pool1d(x.pow(self.p), (x.size(-1))).pow(1. / self.p)\n\n def __repr__(self):\n return self.__class__.__name__ + \\\n '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n ', ' + 'eps=' + str(self.eps) + ')'\nSimilarly, in another competition, I saw a solution that used attention pooling, so I implemented it and got good results. I'm writing my own interpretation based on past commentary and papers.
", "rawMarkdown": "The ideas for the model definition are based on excellent solutions and explanations from participants in past competitions.\n\nFor example, GeM is often used as a pooling trend, so I implemented and experimented with it in this competition. (However, the results were not good, so I did not adopt it.)\n```\nclass GeM(nn.Module):\n def __init__(self, p=3, eps=1e-6):\n super(GeM, self).__init__()\n self.p = nn.Parameter(torch.ones(1) * p)\n self.eps = eps\n\n def forward(self, x):\n x = torch.clamp(x, min=self.eps)\n return F.avg_pool1d(x.pow(self.p), (x.size(-1))).pow(1. / self.p)\n\n def __repr__(self):\n return self.__class__.__name__ + \\\n '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n ', ' + 'eps=' + str(self.eps) + ')'\n```\nSimilarly, in another competition, I saw a solution that used attention pooling, so I implemented it and got good results. I'm writing my own interpretation based on past commentary and papers.", "votes": 1 }, { "id": 3398646, "postDate": "2026-01-29T13:02:56.907Z", "content": "Thank you for sharing — it was very inspiring for me.🥳
", "rawMarkdown": "Thank you for sharing — it was very inspiring for me.🥳", "votes": 1 } ] } ] }, { "id": 3398519, "postDate": "2026-01-29T08:57:48.050Z", "rawMarkdown": "", "isDeleted": true } ], "comments": [ { "id": 3398520, "author_name": "water joe", "author_url": "", "post_date": "2026-01-29T08:58:58.163000", "content": "Amazing😶I think what you did is quite reasonable. Why did you lose so much? Was it because the final submitted model was too small to pull back the 0.8 impact of yours?
", "votes": 1, "replies": [ { "id": 3398533, "author_name": "yukiZ", "author_url": "", "post_date": "2026-01-29T09:28:16.023000", "content": "Hi! Thank you for your comment.
\nThe issue I was unable to solve was that I was unable to implement an appropriate validation strategy.\nThe division was performed using StratifiedGroupKFold, but as shown in the table above, the Fold1 Avg R² was very low at 0.4397, and the variance between folds was also large.
\nI am aware of this issue, and have considered the following solutions:
\nUnfortunately, I didn't have the time to work on these experiments, which I think is one of the reasons why I failed.
", "votes": 1, "replies": [ { "id": 3398549, "author_name": "", "author_url": "", "post_date": "2026-01-29T10:01:30.457000", "content": "", "votes": 0, "replies": [] }, { "id": 3398550, "author_name": "water joe", "author_url": "", "post_date": "2026-01-29T10:02:10.380000", "content": "StratifiedGroupKFold(random_state=82947501)、groups = df_wide['Sampling_Date']、y_stratify = df_wide['State'] —— didn't these work for you? They seemed to have a slight effect in my experiments：
\nFold 0: Global R² = 0.7916, Avg R² = 0.7155
\nFold 1: Global R² = 0.7782, Avg R² = 0.6266
\nFold 2: Global R² = 0.8941, Avg R² = 0.8206
\nFold 3: Global R² = 0.8060, Avg R² = 0.6990
\nFold 4: Global R² = 0.8405, Avg R² = 0.6748
\nI also think discarding a fold is a risky move — even if it's a reluctant choice. If all the models you trained end up discarding the same fold, you’ll definitely lose some valuable information. That’s why I opted for a slight weighting operation instead.
", "votes": 1, "replies": [] }, { "id": 3398559, "author_name": "yukiZ", "author_url": "", "post_date": "2026-01-29T10:19:20.040000", "content": "Wow, that's a pretty good result!\nYes, I'm also using the Split in the public notebook you mentioned(SGKF).\nHowever, I still think that the difference in average R² score between folds of around 0.2 is a significant issue.
\nYou're right. I don't like the Fold Selection approach either, as it increases the risk of overfitting to LB.
", "votes": 1, "replies": [] }, { "id": 3398593, "author_name": "Quan Vu", "author_url": "", "post_date": "2026-01-29T11:03:34.773000", "content": "In this competition, I use a very simple validation strategy: stratified by State and choosing a seed so that the mean and standard deviation of the labels are similar across all 5 folds. There are a few main reasons: first, the dataset is too small, so if we split it in an extreme way, such as grouping by date or by state, the label distribution would become heavily distorted. Second, the host has confirmed that there are no new species in the public and private test sets. Some states have a certain species while others don’t. Grouping by date also risks missing some species or not having all four seasons represented in the training set. As I understand it, each date group corresponds to a trip to a particular farm to collect samples, and it’s possible that some clover species exist only in certain farms.
\nBasically, my split is almost certainly suffering from data leakage, and I accept that. To mitigate this, I use the CV–LB correlation and the LB score to evaluate the model.
", "votes": 2, "replies": [] }, { "id": 3398620, "author_name": "water joe", "author_url": "", "post_date": "2026-01-29T12:21:39.963000", "content": "I also want to ask you how you designed the network architecture. It seems that this is quite uncommon. Regarding this \"diff\", was this idea derived from a paper, or was it just a sudden inspiration? I feel that I don't have much inspiration when it comes to design.
", "votes": 1, "replies": [] }, { "id": 3398623, "author_name": "yukiZ", "author_url": "", "post_date": "2026-01-29T12:26:41.397000", "content": "Thank you for your advice. And congratulations on getting second place, that's amazing!\nAs a \"seed search\" approach, searching to minimize variance between folds seems important for this dataset.
", "votes": 1, "replies": [] }, { "id": 3398629, "author_name": "yukiZ", "author_url": "", "post_date": "2026-01-29T12:44:27.817000", "content": "The ideas for the model definition are based on excellent solutions and explanations from participants in past competitions.
\nFor example, GeM is often used as a pooling trend, so I implemented and experimented with it in this competition. (However, the results were not good, so I did not adopt it.)
\nclass GeM(nn.Module):\n def __init__(self, p=3, eps=1e-6):\n super(GeM, self).__init__()\n self.p = nn.Parameter(torch.ones(1) * p)\n self.eps = eps\n\n def forward(self, x):\n x = torch.clamp(x, min=self.eps)\n return F.avg_pool1d(x.pow(self.p), (x.size(-1))).pow(1. / self.p)\n\n def __repr__(self):\n return self.__class__.__name__ + \\\n '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n ', ' + 'eps=' + str(self.eps) + ')'\nSimilarly, in another competition, I saw a solution that used attention pooling, so I implemented it and got good results. I'm writing my own interpretation based on past commentary and papers.
", "votes": 1, "replies": [] }, { "id": 3398646, "author_name": "water joe", "author_url": "", "post_date": "2026-01-29T13:02:56.907000", "content": "Thank you for sharing — it was very inspiring for me.🥳
", "votes": 1, "replies": [] } ] } ] }, { "id": 3398519, "author_name": "", "author_url": "", "post_date": "2026-01-29T08:57:48.050000", "content": "", "votes": 0, "replies": [] } ], "raw_markdown_by_id": { "3398370": "It seems that a big shake occurred as a result.The shake may have been expected, as there were many well-known Kagglers with fewer subs in the discussions and LB.\nDue to personal reasons, I was only able to participate for two weeks, but the problem setting was simple, and the relationships between the subjects allowed for a variety of approaches, so it was a fun competition, and I would like to express my gratitude to the organizers.\n\n---\n\n# 🔶TLDR\n\n* Basically, i focused on the weaknesses of the models in excellent published NBs and made improvements, and i will publish the main effective changes as solutions.\n * **(1) Regression model with Attention Pooling and MambaBlock.**※Weights are updated using EMAv2 for improved versatility.\n * **(2)Regression model with FiLM, aiming to strengthen feature fusion(Left + Right + Diff + Prod) and eliminate the bottleneck of the regression head**\n * **(3)A regression model using google-siglip to extract features from images.** ※I used the public note book as is for this, so I will skip the explanation below.\n * **(4)A very simple post-process using \"Dry_Clover_g\"*0.8**\n\n---\n\n# 🔶(1) Regression model with Attention Pooling and MambaBlock.\n* Key improvements over the public notebook include.\n* The image also contains dirt, which can be considered noise.\nFor this reason, we adopted Attention Pooling, which can extract only important patches from the feature map extracted by the backbone. The code is below.\n* This also boosts CV/LB and works well, which was a good impression.\n```\nclass AttentionPool1d(nn.Module):\n def __init__(self, in_features, hidden_dim=None):\n super().__init__()\n h_dim = hidden_dim or (in_features // 2)\n \n self.att_net = nn.Sequential(\n nn.Conv1d(in_features, h_dim, kernel_size=1),\n nn.Tanh(),\n nn.Conv1d(h_dim, 1, kernel_size=1)\n )\n \n def forward(self, x):\n scores = self.att_net(x) \n weights = F.softmax(scores, dim=-1) \n return (x * weights).sum(dim=-1)\n\nclass BiomassModelv2(nn.Module):\n def __init__(self, model_name, pretrained=True, backbone_path=None):\n super().__init__()\n self.model_name = model_name\n self.backbone_path = backbone_path\n \n # 1. Load Backbone\n self.backbone = timm.create_model(self.model_name, pretrained=False, num_classes=0, global_pool='')\n \n if hasattr(self.backbone, 'set_grad_checkpointing'):\n self.backbone.set_grad_checkpointing(True)\n print(\"✓ Gradient Checkpointing enabled\")\n \n nf = self.backbone.num_features\n \n # 2. Mamba Fusion\n self.fusion = nn.Sequential(\n LocalMambaBlock(nf, kernel_size=5, dropout=0.1),\n LocalMambaBlock(nf, kernel_size=5, dropout=0.1)\n )\n \n # 3. Pooling (Attention Pooling)\n self.pool = AttentionPool1d(in_features=nf)\n \n # 4. Heads\n self.head_green_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n self.head_clover_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n self.head_dead_raw = nn.Sequential(\n nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n nn.Linear(nf//2, 1), nn.Softplus()\n )\n \n if pretrained:\n self.load_pretrained()\n \n def load_pretrained(self):\n try:\n if self.backbone_path and os.path.exists(self.backbone_path):\n print(f\"Loading backbone weights from local file: {self.backbone_path}\")\n sd = torch.load(self.backbone_path, map_location='cpu')\n if 'model' in sd: sd = sd['model']\n elif 'state_dict' in sd: sd = sd['state_dict']\n else:\n print(\"Downloading backbone weights...\")\n sd = timm.create_model(self.model_name, pretrained=True, num_classes=0, global_pool='').state_dict()\n \n self.backbone.load_state_dict(sd, strict=False)\n print('Pretrained weights loaded.')\n except Exception as e:\n print(f'Warning: pretrained load failed: {e}')\n \n def forward(self, left, right):\n # 1. Extract & Concat\n x_l = self.backbone(left)\n x_r = self.backbone(right)\n x_cat = torch.cat([x_l, x_r], dim=1) # (B, 2N, D)\n \n # 2. Fusion\n x_fused = self.fusion(x_cat) # (B, 2N, D)\n \n # 3. Attention Pooling\n x_pool = self.pool(x_fused.transpose(1, 2))\n \n # 4. Prediction Heads\n green = self.head_green_raw(x_pool)\n clover = self.head_clover_raw(x_pool)\n dead = self.head_dead_raw(x_pool)\n \n # Summation logic\n gdm = green + clover\n total = gdm + dead\n \n return total, gdm, green, clover, dead\n```\n| Fold | Global R² | Avg R² |\n| --- | --- | --- |\nFold 0|0.7440|0.6160\nFold 1|0.7007|0.4397\nFold 2|0.8296|0.7280\nFold 3|0.7196|0.5279\nFold 4|0.8055|0.6567\n\n* The DA set is common, but I had the impression that ShiftScaleRotate worked well.\n```\ndef get_train_transforms():\n return A.Compose([\n A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n A.HorizontalFlip(p=0.5),\n A.VerticalFlip(p=0.5),\n A.RandomRotate90(p=0.5),\n A.Rotate(limit=(-10, 10), p=0.3, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101),\n A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.05, p=0.5),\n A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n A.RandomBrightnessContrast(brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2), p=0.3),\n A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.3, rotate_limit=15, border_mode=0, p=0.50),\n ToTensorV2()\n ], p=1.0)\n```\n* Finally, to make the model more versatile, we used EMAv2 (EMAv3 is the latest, but we used EMAv2) with Decay 0.999. This generally reduces CV, but increases versatility, improving LB.\n```\n(e.g.)\nfrom timm.utils import ModelEmaV2\nema = ModelEmaV2(base_model, decay=0.999)\n```\n\n---\n\n# 🔶(2)Regression model with FiLM, aiming to strengthen feature fusion(Left + Right + Diff + Prod) and eliminate the bottleneck of the regression head\n* Key improvements over the public notebook include.\n* We also made some enhancements to the FiIM layer, including a network that generates modulation parameters from the context.\n```\nclass FiLMV2(nn.Module):\n def __init__(self, feat_dim):\n super().__init__()\n self.mlp = nn.Sequential(\n nn.Linear(feat_dim, feat_dim), \n nn.ReLU(inplace=True), \n nn.Linear(feat_dim, feat_dim * 2)\n )\n\n def forward(self, context):\n gamma_beta = self.mlp(context)\n return torch.chunk(gamma_beta, 2, dim=1)\nclass MultiTargetRegressorV2(nn.Module):\n def __init__(self, model_name, pretrained=True, num_classes=3, dropout=0.2, freeze_backbone=False):\n super().__init__()\n self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='avg')\n self.feat_dim = self.backbone.num_features \n self.film = FiLMV2(self.feat_dim)\n self.dropout = nn.Dropout(dropout)\n\n # Left + Right + Diff + Prod\n fusion_dim = self.feat_dim * 4\n\n def make_head():\n return nn.Sequential(\n nn.Linear(fusion_dim, 512),\n nn.BatchNorm1d(512),\n nn.ReLU(inplace=True),\n nn.Dropout(dropout),\n nn.Linear(512, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, 1),\n )\n\n self.head_green = make_head()\n self.head_clover = make_head()\n self.head_dead = make_head()\n\n self.softplus = nn.Softplus(beta=1.0)\n\n if freeze_backbone:\n for p in self.backbone.parameters():\n p.requires_grad = False\n\n def forward(self, left_img, right_img):\n left_feat = self.backbone(left_img)\n right_feat = self.backbone(right_img)\n\n context = (left_feat + right_feat) / 2\n gamma, beta = self.film(context)\n\n left_feat_mod = left_feat * (1 + gamma) + beta\n right_feat_mod = right_feat * (1 + gamma) + beta\n\n diff_feat = torch.abs(left_feat_mod - right_feat_mod)\n prod_feat = left_feat_mod * right_feat_mod\n\n combined = torch.cat([left_feat_mod, right_feat_mod, diff_feat, prod_feat], dim=1)\n\n green = self.softplus(self.head_green(combined)) \n clover = self.softplus(self.head_clover(combined))\n dead = self.softplus(self.head_dead(combined)) \n\n logits = torch.cat([green, clover, dead], dim=1)\n\n return logits\n```\n* As mentioned above, OnlineAug's ShiftScaleRotate (called RandomAffine in torchvision) seemed to work well.\n\n```\n transform = transforms.Compose([\n transforms.Resize(target_image_size),\n transforms.RandomHorizontalFlip(p=0.5),\n transforms.RandomVerticalFlip(p=0.5),\n transforms.RandomApply([transforms.ColorJitter(\n brightness=0.1, \n contrast=0.1, \n saturation=0.1, \n hue=0.05)], p=0.5),\n \n transforms.RandomApply([transforms.RandomRotation(\n degrees=10, \n interpolation=transforms.InterpolationMode.BILINEAR\n )], p=0.3),\n \n transforms.RandomApply([transforms.RandomAffine(\n degrees=15,\n translate=(0.1, 0.1),\n scale=(0.7, 1.3),\n interpolation=transforms.InterpolationMode.BILINEAR\n )], p=0.5),\n transforms.RandomApply([transforms.RandomChoice([\n transforms.GaussianBlur(kernel_size=5),\n transforms.ElasticTransform(alpha=50.0),\n transforms.RandomPerspective(distortion_scale=0.5, p=1.0) \n ])], p=0.3),\n \n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n```\n\n| Fold | weighted_r2 |\n| --- | --- |\nFold1|0.7996\nFold2|0.7600\nFold3|0.7161\nFold4|0.6141\nFold5|0.7435\n\n* As an aside, I was also considering using EMA here, but since the computational resources were being used for other tasks, I decided not to use it. I think that CV will probably decrease, but LB will increase.\n\n---\n\n# 🔶(4)A very simple post-process using \"Dry_Clover_g\"*0.8\n* Honestly, I'm not sure why this worked. I wonder if there was data drift in the Train/Test (LB/PB) run. LB/PB showed a large boost of approximately 0.01.\n```\n(e.g.)\nsubmission.loc[submission['sample_id'].str.endswith('Dry_Clover_g'),'target']*=0.8\nsubmission.to_csv(OUTPUT_FILE, index=False)\n```\n\n", "3398520": "Amazing😶I think what you did is quite reasonable. Why did you lose so much? Was it because the final submitted model was too small to pull back the 0.8 impact of yours?", "3398519": "" } }