{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":711363,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":540463,"modelId":553659}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":823.449492,"end_time":"2026-01-03T04:06:15.662245","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-01-03T03:52:32.212753","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n\n* ConvNext-Tiny  Backbone  , Attention layers , Attention Blocks\n\n* 4  learning Rate  1)Backbone , 2) attention, layer , 3)attention head , 4)Projection layer \n","metadata":{"papermill":{"duration":0.004132,"end_time":"2026-01-03T03:52:34.809357","exception":false,"start_time":"2026-01-03T03:52:34.805225","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Import Packages","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom glob import glob\nimport os\nimport cv2\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n\nimport torch\nfrom torch.utils.data import Dataset , DataLoader\n\nimport torchvision.transforms as T\n\nimport torch.nn as nn\n\nimport torch.nn.functional as F\n\nimport timm\nimport torchvision.models  as models\nimport torch.optim as optim\n","metadata":{"papermill":{"duration":9.825196,"end_time":"2026-01-03T03:52:44.637942","exception":false,"start_time":"2026-01-03T03:52:34.812746","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:41.420036Z","iopub.execute_input":"2026-01-07T20:33:41.420372Z","iopub.status.idle":"2026-01-07T20:33:41.425374Z","shell.execute_reply.started":"2026-01-07T20:33:41.420347Z","shell.execute_reply":"2026-01-07T20:33:41.424768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n# Config","metadata":{"papermill":{"duration":0.003079,"end_time":"2026-01-03T03:52:44.644306","exception":false,"start_time":"2026-01-03T03:52:44.641227","status":"completed"},"tags":[]}},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\ndevice","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:10:00.447281Z","iopub.execute_input":"2026-01-07T20:10:00.447570Z","iopub.status.idle":"2026-01-07T20:10:00.452261Z","shell.execute_reply.started":"2026-01-07T20:10:00.447545Z","shell.execute_reply":"2026-01-07T20:10:00.451528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    \n    data_dir=\"/kaggle/input/csiro-biomass\"\n    \n    train_csv=pd.read_csv(os.path.join(data_dir,\"train.csv\"))\n    test_csv=os.path.join(data_dir,\"test.csv\")\n    train_images=os.path.join(data_dir,\"train\")\n    test_images=os.path.join(data_dir,\"test\")\n    batch=8\n    \n    epochs=35\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n\n#Metric Score \ndef weighted_r2_score(y_true, y_pred):\n        weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n        r2s = []\n    \n        for i in range(5):\n            yt, yp = y_true[:, i], y_pred[:, i]\n            ss_res = np.sum((yt - yp) ** 2)\n            ss_tot = np.sum((yt - yt.mean()) ** 2)\n            r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0\n            r2s.append(r2)\n    \n        r2s = np.array(r2s)\n        return np.sum(r2s * weights) / weights.sum(), r2s","metadata":{"papermill":{"duration":0.029847,"end_time":"2026-01-03T03:52:44.677308","exception":false,"start_time":"2026-01-03T03:52:44.647461","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:49.984800Z","iopub.execute_input":"2026-01-07T20:33:49.985111Z","iopub.status.idle":"2026-01-07T20:33:50.000127Z","shell.execute_reply.started":"2026-01-07T20:33:49.985088Z","shell.execute_reply":"2026-01-07T20:33:49.999555Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Seting Up Data","metadata":{}},{"cell_type":"code","source":"train_wide = Config.train_csv.pivot_table(\n    index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'],\n    columns='target_name',\n    values='target',\n    aggfunc='first'\n).reset_index()\n\n\ntrain_wide[\"full_path\"]=train_wide[\"image_path\"].apply(lambda x: os.path.join(Config.data_dir,x))\nos.path.exists(train_wide.full_path.iloc[0])","metadata":{"papermill":{"duration":0.042214,"end_time":"2026-01-03T03:52:44.758287","exception":false,"start_time":"2026-01-03T03:52:44.716073","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:51.551600Z","iopub.execute_input":"2026-01-07T20:33:51.552363Z","iopub.status.idle":"2026-01-07T20:33:51.570688Z","shell.execute_reply.started":"2026-01-07T20:33:51.552331Z","shell.execute_reply":"2026-01-07T20:33:51.570068Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Loading Sample","metadata":{"papermill":{"duration":0.003616,"end_time":"2026-01-03T03:52:44.786852","exception":false,"start_time":"2026-01-03T03:52:44.783236","status":"completed"},"tags":[]}},{"cell_type":"code","source":"x=cv2.imread(train_wide.full_path.iloc[6])\nx=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)\n\nplt.imshow(x)\nplt.axis(\"off\")\nplt.show()","metadata":{"papermill":{"duration":0.467317,"end_time":"2026-01-03T03:52:45.273541","exception":false,"start_time":"2026-01-03T03:52:44.806224","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:51.866002Z","iopub.execute_input":"2026-01-07T20:33:51.866846Z","iopub.status.idle":"2026-01-07T20:33:52.068754Z","shell.execute_reply.started":"2026-01-07T20:33:51.866813Z","shell.execute_reply":"2026-01-07T20:33:52.067921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_paths=train_wide[\"full_path\"].values\n#target_cols = [\"Dry_Clover_g\",\"Dry_Dead_g\",\"Dry_Green_g\",\"Dry_Total_g\",\"GDM_g\"]\n\n","metadata":{"papermill":{"duration":0.01336,"end_time":"2026-01-03T03:52:45.294994","exception":false,"start_time":"2026-01-03T03:52:45.281634","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:52.069974Z","iopub.execute_input":"2026-01-07T20:33:52.070230Z","iopub.status.idle":"2026-01-07T20:33:52.073816Z","shell.execute_reply.started":"2026-01-07T20:33:52.070208Z","shell.execute_reply":"2026-01-07T20:33:52.073159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_cols = [\"Dry_Clover_g\",\"Dry_Dead_g\",\"Dry_Green_g\",\"Dry_Total_g\",\"GDM_g\"]\ny = train_wide[target_cols].values.astype(\"float32\")","metadata":{"papermill":{"duration":0.013966,"end_time":"2026-01-03T03:52:45.316274","exception":false,"start_time":"2026-01-03T03:52:45.302308","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:53.215631Z","iopub.execute_input":"2026-01-07T20:33:53.216457Z","iopub.status.idle":"2026-01-07T20:33:53.220802Z","shell.execute_reply.started":"2026-01-07T20:33:53.216426Z","shell.execute_reply":"2026-01-07T20:33:53.220078Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset & Data Loader ","metadata":{"papermill":{"duration":0.007197,"end_time":"2026-01-03T03:52:45.331151","exception":false,"start_time":"2026-01-03T03:52:45.323954","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"I loaded the Data by Open Cv2 and transfer it to ToPILImage at the Transform ","metadata":{}},{"cell_type":"code","source":"class Biomass(Dataset):\n    def __init__(self,paths,labels,transform=None):\n        self.paths=paths\n        self.labels=labels\n        self.transform=transform\n\n\n    def __len__(self):\n        l=len((self.paths))\n        return l\n\n\n    def __getitem__(self,idx):\n        img=cv2.imread(self.paths[idx])\n        img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n        img=self.transform(img) if self.transform else img\n        label=torch.tensor(self.labels[idx],dtype=torch.float32)\n        return img , label\n        \n\n","metadata":{"papermill":{"duration":0.01436,"end_time":"2026-01-03T03:52:45.352721","exception":false,"start_time":"2026-01-03T03:52:45.338361","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:53.842671Z","iopub.execute_input":"2026-01-07T20:33:53.843101Z","iopub.status.idle":"2026-01-07T20:33:53.848743Z","shell.execute_reply.started":"2026-01-07T20:33:53.843069Z","shell.execute_reply":"2026-01-07T20:33:53.847852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torchvision.transforms as T\n\nimg_size = 224\n\ntrain_transform = T.Compose([\n    T.ToPILImage(),                          # from cv2 (numpy) -> PIL\n    T.Resize((224, 224)),\n    T.RandomHorizontalFlip(),\n    T.RandomVerticalFlip(),\n    T.ColorJitter(0.1, 0.1, 0.1, 0.05),\n    T.ToTensor(),\n    T.Normalize([0.485, 0.456, 0.406],\n                         [0.229, 0.224, 0.225])\n])\n\nvalid_transform = T.Compose([\n    T.ToPILImage(),\n    T.Resize((224, 224)),\n    T.ToTensor(),\n    T.Normalize([0.485, 0.456, 0.406],\n                         [0.229, 0.224, 0.225])\n])\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:54.030002Z","iopub.execute_input":"2026-01-07T20:33:54.030257Z","iopub.status.idle":"2026-01-07T20:33:54.035607Z","shell.execute_reply.started":"2026-01-07T20:33:54.030235Z","shell.execute_reply":"2026-01-07T20:33:54.034884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split \n\nX_train,X_valid,y_train,y_valid=train_test_split(image_paths,y,test_size=0.2,random_state=44)\n\ntrain_dataset = Biomass(X_train, y_train, transform=train_transform)\nvalid_dataset = Biomass(X_valid, y_valid, transform=valid_transform)\n\ntrain_loader=DataLoader(train_dataset,batch_size=Config.batch,shuffle=True)\nvalid_loader=DataLoader(valid_dataset,batch_size=Config.batch ,shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:54.320586Z","iopub.execute_input":"2026-01-07T20:33:54.320879Z","iopub.status.idle":"2026-01-07T20:33:54.327070Z","shell.execute_reply.started":"2026-01-07T20:33:54.320855Z","shell.execute_reply":"2026-01-07T20:33:54.326474Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Attention ConvNext Model Set Up ","metadata":{"papermill":{"duration":0.007383,"end_time":"2026-01-03T03:52:45.367426","exception":false,"start_time":"2026-01-03T03:52:45.360043","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Attention Layers and attention head  + Convex Tiny ","metadata":{}},{"cell_type":"code","source":"\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport timm\n\nclass ConvNeXtWithAttention(nn.Module):\n    def __init__(\n        self,\n        backbone_name=\"convnext_tiny\",\n        pretrained=True,\n        embed_dim=256,\n        num_attn_layers=2,\n        num_heads=8,\n        dropout=0.05,\n        out_dim=5,  # regression\n    ):\n        super().__init__()\n\n        # 1) ConvNeXt backbone -> feature map (B, C, H, W)\n        self.backbone = timm.create_model(\n            backbone_name,\n            pretrained=pretrained,\n            features_only=True,\n            out_indices=(3,)  # last stage\n        )\n        in_ch = self.backbone.feature_info.channels()[0]\n\n        # 2) Project channels to transformer embed dim\n        self.proj = nn.Conv2d(in_ch, embed_dim, kernel_size=1)\n\n        # 3) Stack of Transformer Encoder layers (self-attention)\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=embed_dim,\n            nhead=num_heads,\n            dim_feedforward=embed_dim * 4,\n            dropout=dropout,\n            batch_first=True,\n            activation=\"gelu\",\n            norm_first=True,\n        )\n        self.attn = nn.TransformerEncoder(encoder_layer, num_layers=num_attn_layers)\n\n        # 4) Regression head\n        self.head = nn.Sequential(\n            nn.LayerNorm(embed_dim),\n            nn.Linear(embed_dim, embed_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(embed_dim, out_dim),\n        )\n\n    def forward(self, x):\n        # x: (B, 3, H, W)\n        feat = self.backbone(x)[0]         # (B, C, Hf, Wf)\n        feat = self.proj(feat)             # (B, E, Hf, Wf)\n\n        B, E, Hf, Wf = feat.shape\n        tokens = feat.flatten(2).transpose(1, 2)  # (B, N, E) where N=Hf*Wf\n\n        tokens = self.attn(tokens)         # (B, N, E)\n\n        # Global average over tokens\n        pooled = tokens.mean(dim=1)        # (B, E)\n\n        return self.head(pooled) # (B,)\n\nmodelx= ConvNeXtWithAttention(\n    backbone_name=\"convnext_tiny\",\n    pretrained=True ,\n    embed_dim=256,\n    num_attn_layers=2,\n    num_heads=8,\n    out_dim=5\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:33:56.311301Z","iopub.execute_input":"2026-01-07T20:33:56.311590Z","iopub.status.idle":"2026-01-07T20:33:57.015320Z","shell.execute_reply.started":"2026-01-07T20:33:56.311564Z","shell.execute_reply":"2026-01-07T20:33:57.014577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"criterion=nn.SmoothL1Loss()\n#optimizer=torch.optim.AdamW(model.parameters(),lr=3e-4,weight_decay=1e-2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:10:41.547661Z","iopub.execute_input":"2026-01-07T20:10:41.548327Z","iopub.status.idle":"2026-01-07T20:10:41.551658Z","shell.execute_reply.started":"2026-01-07T20:10:41.548298Z","shell.execute_reply":"2026-01-07T20:10:41.550978Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"I used different learning rates for the backbone, attention layers, and attention head.","metadata":{}},{"cell_type":"code","source":"import torch.optim as optim\noptimizer = optim.AdamW([\n    {\"params\": modelx.backbone.parameters(), \"lr\": 3e-6},\n    {\"params\": modelx.proj.parameters(), \"lr\": 1e-4},\n    {\"params\": modelx.attn.parameters(), \"lr\": 1e-4},\n    {\"params\": modelx.head.parameters(), \"lr\": 1e-4},\n], weight_decay=0.05)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:34:04.372172Z","iopub.execute_input":"2026-01-07T20:34:04.372935Z","iopub.status.idle":"2026-01-07T20:34:04.379446Z","shell.execute_reply.started":"2026-01-07T20:34:04.372895Z","shell.execute_reply":"2026-01-07T20:34:04.378855Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodelx = modelx.to(device)\n\nfor epoch in range(Config.epochs):\n    modelx.train()\n    total_loss=0\n    for  img , label in train_loader:\n        img,label=img.to(device),label.to(device)\n        preds=modelx(img)\n        loss=criterion(preds,label)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        total_loss+=loss.item()\n#        scheduler.step() \n\n    avg_loss = total_loss / len(train_loader)\n    print(f\"Epoch {epoch+1}/{Config.epochs} Loss: {avg_loss:.4f}\")\n        \n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:34:12.976897Z","iopub.execute_input":"2026-01-07T20:34:12.977500Z","iopub.status.idle":"2026-01-07T20:48:58.051722Z","shell.execute_reply.started":"2026-01-07T20:34:12.977472Z","shell.execute_reply":"2026-01-07T20:48:58.051031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"modelx.eval()\nall_preds, all_labels = [], []\n\nwith torch.no_grad():\n    for imgs, labels in valid_loader:\n        imgs = imgs.to(device)\n\n        preds = modelx(imgs).detach().cpu().numpy()\n        labels = labels.detach().cpu().numpy()\n\n        all_preds.append(preds)\n        all_labels.append(labels)\n\nall_preds = np.vstack(all_preds)\nall_labels = np.vstack(all_labels)\n\nscore, per_label_r2 = weighted_r2_score(all_labels, all_preds)\nprint(\"Weighted R2:\", score)\nprint(\"Per-target R2:\", per_label_r2)","metadata":{"papermill":{"duration":0.010409,"end_time":"2026-01-03T04:06:13.020100","exception":false,"start_time":"2026-01-03T04:06:13.009691","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:48:58.052886Z","iopub.execute_input":"2026-01-07T20:48:58.053129Z","iopub.status.idle":"2026-01-07T20:49:02.164103Z","shell.execute_reply.started":"2026-01-07T20:48:58.053101Z","shell.execute_reply":"2026-01-07T20:49:02.163447Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Saving The model","metadata":{}},{"cell_type":"code","source":"import torch\n\nsave_path = \"/kaggle/working/convnext_attn_drop_0.05_35_epoch.pth\"\n\ntorch.save(\n    {\n        \"model_state\": modelx.state_dict(),\n        \"config\": {\n            \"backbone_name\": \"convnext_tiny\",\n            \"embed_dim\": 256,\n            \"num_attn_layers\": 2,\n            \"num_heads\": 8,\n            \"out_dim\": 5,\n        },\n    },\n    save_path,\n)\n\nprint(\"Saved to:\", save_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-07T20:49:51.531959Z","iopub.execute_input":"2026-01-07T20:49:51.532654Z","iopub.status.idle":"2026-01-07T20:49:51.670478Z","shell.execute_reply.started":"2026-01-07T20:49:51.532626Z","shell.execute_reply":"2026-01-07T20:49:51.669744Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Then I load the model at the inference to make the Submission ","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}