# %% [markdown] # # Workflow # 1. Load Nerual network. # 2. Use neural network to look at the images and train it for each numerical values # 3. Append the values for the test values # 4. Use regression model on the output of the neural network # # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:00.517086Z","iopub.execute_input":"2025-11-10T02:05:00.517608Z","iopub.status.idle":"2025-11-10T02:05:00.521279Z","shell.execute_reply.started":"2025-11-10T02:05:00.517586Z","shell.execute_reply":"2025-11-10T02:05:00.520658Z"}} import pandas as pd import numpy as np import matplotlib.pyplot as plt from PIL import Image # %% [code] # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:00.522636Z","iopub.execute_input":"2025-11-10T02:05:00.522841Z","iopub.status.idle":"2025-11-10T02:05:00.565071Z","shell.execute_reply.started":"2025-11-10T02:05:00.522825Z","shell.execute_reply":"2025-11-10T02:05:00.564305Z"}} train_df = pd.read_csv("/kaggle/input/dataset/train.csv") test_df = pd.read_csv("/kaggle/input/dataset/test.csv") # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:00.565865Z","iopub.execute_input":"2025-11-10T02:05:00.566118Z","iopub.status.idle":"2025-11-10T02:05:00.573952Z","shell.execute_reply.started":"2025-11-10T02:05:00.566094Z","shell.execute_reply":"2025-11-10T02:05:00.573241Z"}} train_df['target_name'].value_counts() # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:00.575397Z","iopub.execute_input":"2025-11-10T02:05:00.575763Z","iopub.status.idle":"2025-11-10T02:05:00.592757Z","shell.execute_reply.started":"2025-11-10T02:05:00.575744Z","shell.execute_reply":"2025-11-10T02:05:00.592014Z"}} dry_clover_g = train_df[train_df['target_name'] == "Dry_Clover_g"] dry_dead_g = train_df[train_df['target_name'] == "Dry_Dead_g"] dry_green_g = train_df[train_df['target_name'] == "Dry_Green_g"] dry_total_g = train_df[train_df['target_name'] == "Dry_Total_g"] gdm_g = train_df[train_df['target_name'] == "GDM_g"] # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:00.593484Z","iopub.execute_input":"2025-11-10T02:05:00.593703Z","iopub.status.idle":"2025-11-10T02:05:00.619314Z","shell.execute_reply.started":"2025-11-10T02:05:00.593682Z","shell.execute_reply":"2025-11-10T02:05:00.618715Z"}} dry_total_g # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:00.620676Z","iopub.execute_input":"2025-11-10T02:05:00.620858Z","iopub.status.idle":"2025-11-10T02:05:00.631495Z","shell.execute_reply.started":"2025-11-10T02:05:00.620843Z","shell.execute_reply":"2025-11-10T02:05:00.630776Z"}} dry_total_g['Species'].value_counts() # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:00.632158Z","iopub.execute_input":"2025-11-10T02:05:00.632391Z","iopub.status.idle":"2025-11-10T02:05:07.003380Z","shell.execute_reply.started":"2025-11-10T02:05:00.632377Z","shell.execute_reply":"2025-11-10T02:05:07.002784Z"}} import torch import torchvision from torchvision import datasets, transforms import torch.nn as nn from torchvision.transforms import ToTensor # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.004000Z","iopub.execute_input":"2025-11-10T02:05:07.004322Z","iopub.status.idle":"2025-11-10T02:05:07.014333Z","shell.execute_reply.started":"2025-11-10T02:05:07.004304Z","shell.execute_reply":"2025-11-10T02:05:07.013403Z"}} dry_total_g_height = dry_total_g.drop(['Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'target_name', 'target'], axis=1) dry_total_g_height # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.015201Z","iopub.execute_input":"2025-11-10T02:05:07.015751Z","iopub.status.idle":"2025-11-10T02:05:07.031907Z","shell.execute_reply.started":"2025-11-10T02:05:07.015734Z","shell.execute_reply":"2025-11-10T02:05:07.031135Z"}} #Define transforms (resize, normalize) transform = transforms.Compose([ transforms.Resize((224,224)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225]) ]) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.032637Z","iopub.execute_input":"2025-11-10T02:05:07.032904Z","iopub.status.idle":"2025-11-10T02:05:07.047357Z","shell.execute_reply.started":"2025-11-10T02:05:07.032873Z","shell.execute_reply":"2025-11-10T02:05:07.046611Z"}} #Training loop from torch.utils.data import DataLoader, Dataset class HeightDataset(Dataset): # Step 1a: Initialize - store dataframe, image directory, and transforms def __init__(self, df, img_dir, transform=None): self.df = df self.img_dir = img_dir self.transform = transform def __len__(self): return len(self.df) #Returns number of rows in df def __getitem__(self, idx): # Get one row from dataframe row = self.df.iloc[idx] # Build full path to image file path = f"{self.img_dir}/{row['image_path']}" #Need to make img_dir # Load image from disk and convert to RGB image = Image.open(path).convert('RGB') # Apply trasnforms (resize to 224x224, normalize, etc) if self.transform: image = self.transform(image) # Get the target value target = torch.tensor(row['Height_Ave_cm'], dtype=torch.float32) # Get sample_id for reference sample_id = row['sample_id'] # Return tuple: (image_tensor, height_value, sample_id) return image, target, sample_id train_dataset = HeightDataset( df=dry_total_g_height, img_dir = '/kaggle/input/dataset/', transform = transform ) train_dataloader = DataLoader( train_dataset, batch_size=32, shuffle=True ) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.049742Z","iopub.execute_input":"2025-11-10T02:05:07.049906Z","iopub.status.idle":"2025-11-10T02:05:07.115181Z","shell.execute_reply.started":"2025-11-10T02:05:07.049893Z","shell.execute_reply":"2025-11-10T02:05:07.114314Z"}} device = 'cuda' if torch.cuda.is_available() else 'cpu' device # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.115775Z","iopub.execute_input":"2025-11-10T02:05:07.115987Z","iopub.status.idle":"2025-11-10T02:05:07.417832Z","shell.execute_reply.started":"2025-11-10T02:05:07.115967Z","shell.execute_reply":"2025-11-10T02:05:07.416957Z"}} # Heigth model from torchvision.models import efficientnet_b0 class height_nn(nn.Module): def __init__(self): super().__init__() self.model = efficientnet_b0(pretrained=False ) #removce the classifier self.model.classifier = nn.Sequential( nn.Linear(1280, 128), nn.ReLU(), nn.Linear(128, 1) ) for param in self.model.parameters(): param.requires_grad = True def forward(self, x): return self.model(x) model_height = height_nn().to(device) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.418704Z","iopub.execute_input":"2025-11-10T02:05:07.418979Z","iopub.status.idle":"2025-11-10T02:05:07.423937Z","shell.execute_reply.started":"2025-11-10T02:05:07.418957Z","shell.execute_reply":"2025-11-10T02:05:07.423298Z"}} print(f"Min height: {dry_total_g_height['Height_Ave_cm'].min()}") print(f"Max height: {dry_total_g_height['Height_Ave_cm'].max()}") print(f"Mean height: {dry_total_g_height['Height_Ave_cm'].mean()}") # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.424593Z","iopub.execute_input":"2025-11-10T02:05:07.424820Z","iopub.status.idle":"2025-11-10T02:05:07.442751Z","shell.execute_reply.started":"2025-11-10T02:05:07.424800Z","shell.execute_reply":"2025-11-10T02:05:07.442088Z"}} dry_total_g_Pre_GSHH_NDVI = dry_total_g.drop(['Sampling_Date', 'State', 'Species', 'Height_Ave_cm', 'target_name', 'target'], axis=1) dry_total_g_Pre_GSHH_NDVI # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.443424Z","iopub.execute_input":"2025-11-10T02:05:07.444084Z","iopub.status.idle":"2025-11-10T02:05:07.455453Z","shell.execute_reply.started":"2025-11-10T02:05:07.444060Z","shell.execute_reply":"2025-11-10T02:05:07.454849Z"}} class NDVIDataset(Dataset): # Step 1a: Initialize - store dataframe, image directory, and transforms def __init__(self, df, img_dir, transform=None): self.df = df self.img_dir = img_dir self.transform = transform def __len__(self): return len(self.df) #Returns number of rows in df def __getitem__(self, idx): # Get one row from dataframe row = self.df.iloc[idx] # Build full path to image file path = f"{self.img_dir}/{row['image_path']}" #Need to make img_dir # Load image from disk and convert to RGB image = Image.open(path).convert('RGB') # Apply trasnforms (resize to 224x224, normalize, etc) if self.transform: image = self.transform(image) # Get the target value target = torch.tensor(row['Pre_GSHH_NDVI'], dtype=torch.float32) # Get sample_id for reference sample_id = row['sample_id'] # Return tuple: (image_tensor, height_value, sample_id) return image, target, sample_id train_dataset = NDVIDataset( df=dry_total_g_Pre_GSHH_NDVI, img_dir = '/kaggle/input/dataset/', transform = transform ) train_dataloader_NDVI = DataLoader( train_dataset, batch_size=32, shuffle=True ) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.456024Z","iopub.execute_input":"2025-11-10T02:05:07.456196Z","iopub.status.idle":"2025-11-10T02:05:07.584349Z","shell.execute_reply.started":"2025-11-10T02:05:07.456183Z","shell.execute_reply":"2025-11-10T02:05:07.583659Z"}} # NDVI model from torchvision.models import efficientnet_b0 class NDVI_nn(nn.Module): def __init__(self): super().__init__() self.model = efficientnet_b0(pretrained=False) #removce the classifier self.model.classifier = nn.Sequential( nn.Linear(1280, 128), nn.ReLU(), nn.Linear(128, 1), nn.Sigmoid() ) for param in self.model.parameters(): param.requires_grad = True def forward(self, x): return self.model(x) model_NDVI = NDVI_nn().to(device) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.584964Z","iopub.execute_input":"2025-11-10T02:05:07.585192Z","iopub.status.idle":"2025-11-10T02:05:07.589724Z","shell.execute_reply.started":"2025-11-10T02:05:07.585170Z","shell.execute_reply":"2025-11-10T02:05:07.588994Z"}} print(f"Min NDVI: {dry_total_g_Pre_GSHH_NDVI['Pre_GSHH_NDVI'].min()}") print(f"Max NDVI: {dry_total_g_Pre_GSHH_NDVI['Pre_GSHH_NDVI'].max()}") print(f"Mean NDVI: {dry_total_g_Pre_GSHH_NDVI['Pre_GSHH_NDVI'].mean()}") # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.590432Z","iopub.execute_input":"2025-11-10T02:05:07.590617Z","iopub.status.idle":"2025-11-10T02:05:07.616159Z","shell.execute_reply.started":"2025-11-10T02:05:07.590602Z","shell.execute_reply":"2025-11-10T02:05:07.615549Z"}} dry_total_g # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:07.616800Z","iopub.execute_input":"2025-11-10T02:05:07.616981Z","iopub.status.idle":"2025-11-10T02:05:08.049561Z","shell.execute_reply.started":"2025-11-10T02:05:07.616967Z","shell.execute_reply":"2025-11-10T02:05:08.048813Z"}} # Classification for the species from sklearn.preprocessing import LabelEncoder dry_total_g_species = dry_total_g.drop(['Sampling_Date', 'State', 'Pre_GSHH_NDVI','Height_Ave_cm', 'target_name', 'target'], axis=1) # Encode Species le = LabelEncoder() species_encoded = le.fit_transform(dry_total_g_species['Species']) dry_total_g_species['Species_encoded'] = le.fit_transform(dry_total_g_species['Species']) species_dict = dry_total_g_species[['Species', 'Species_encoded']].drop_duplicates().set_index('Species')['Species_encoded'].to_dict() sorted_species_dict = sorted(species_dict.items()) sorted_species_dict # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.050219Z","iopub.execute_input":"2025-11-10T02:05:08.050559Z","iopub.status.idle":"2025-11-10T02:05:08.054779Z","shell.execute_reply.started":"2025-11-10T02:05:08.050542Z","shell.execute_reply":"2025-11-10T02:05:08.053932Z"}} #Save the label encoder so i can decode predictions later import pickle pickle.dump(le, open('lable_encoder.pkl', 'wb')) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.055387Z","iopub.execute_input":"2025-11-10T02:05:08.055588Z","iopub.status.idle":"2025-11-10T02:05:08.069223Z","shell.execute_reply.started":"2025-11-10T02:05:08.055570Z","shell.execute_reply":"2025-11-10T02:05:08.068759Z"}} dry_total_g_species_dropped = dry_total_g_species.drop('Species', axis=1) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.069909Z","iopub.execute_input":"2025-11-10T02:05:08.070100Z","iopub.status.idle":"2025-11-10T02:05:08.088171Z","shell.execute_reply.started":"2025-11-10T02:05:08.070085Z","shell.execute_reply":"2025-11-10T02:05:08.087490Z"}} dry_total_g_species_dropped # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.088887Z","iopub.execute_input":"2025-11-10T02:05:08.089135Z","iopub.status.idle":"2025-11-10T02:05:08.102063Z","shell.execute_reply.started":"2025-11-10T02:05:08.089118Z","shell.execute_reply":"2025-11-10T02:05:08.101297Z"}} # Training step class SpeciesDataset(Dataset): # Step 1a: Initialize - store dataframe, image directory, and transforms def __init__(self, df, img_dir, transform=None): self.df = df self.img_dir = img_dir self.transform = transform def __len__(self): return len(self.df) #Returns number of rows in df def __getitem__(self, idx): # Get one row from dataframe row = self.df.iloc[idx] # Build full path to image file path = f"{self.img_dir}/{row['image_path']}" #Need to make img_dir # Load image from disk and convert to RGB image = Image.open(path).convert('RGB') # Apply trasnforms (resize to 224x224, normalize, etc) if self.transform: image = self.transform(image) # Get the target value target = torch.tensor(row['Species_encoded'], dtype=torch.long) # Use long, not float32 # Get sample_id for reference sample_id = row['sample_id'] # Return tuple: (image_tensor, height_value, sample_id) return image, target, sample_id train_dataset = SpeciesDataset( df=dry_total_g_species, img_dir = '/kaggle/input/dataset/', transform = transform ) train_dataloader_species = DataLoader( train_dataset, batch_size=32, shuffle=True ) # %% [markdown] # # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.102801Z","iopub.execute_input":"2025-11-10T02:05:08.103105Z","iopub.status.idle":"2025-11-10T02:05:08.232866Z","shell.execute_reply.started":"2025-11-10T02:05:08.103085Z","shell.execute_reply":"2025-11-10T02:05:08.232143Z"}} # Model for Species class SpeciesNN(nn.Module): def __init__(self, num_classes): super().__init__() self.model = efficientnet_b0(pretrained=False) self.model.classifier = nn.Sequential( nn.Linear(1280, 128), nn.ReLU(), nn.Linear(128, num_classes) ) def forward(self, x): return self.model(x) num_species = len(le.classes_) model_species = SpeciesNN(num_species).to(device) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.236391Z","iopub.execute_input":"2025-11-10T02:05:08.236576Z","iopub.status.idle":"2025-11-10T02:05:08.247879Z","shell.execute_reply.started":"2025-11-10T02:05:08.236561Z","shell.execute_reply":"2025-11-10T02:05:08.247110Z"}} train_df # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.248692Z","iopub.execute_input":"2025-11-10T02:05:08.248896Z","iopub.status.idle":"2025-11-10T02:05:08.266043Z","shell.execute_reply.started":"2025-11-10T02:05:08.248872Z","shell.execute_reply":"2025-11-10T02:05:08.265499Z"}} # Seperate the state dry_total_g_State = dry_total_g.drop(['Sampling_Date', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm', 'target_name', 'target'], axis=1) dry_total_g_State # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.266820Z","iopub.execute_input":"2025-11-10T02:05:08.267309Z","iopub.status.idle":"2025-11-10T02:05:08.280030Z","shell.execute_reply.started":"2025-11-10T02:05:08.267287Z","shell.execute_reply":"2025-11-10T02:05:08.279433Z"}} transform # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.280689Z","iopub.execute_input":"2025-11-10T02:05:08.280994Z","iopub.status.idle":"2025-11-10T02:05:08.299698Z","shell.execute_reply.started":"2025-11-10T02:05:08.280970Z","shell.execute_reply":"2025-11-10T02:05:08.299138Z"}} dry_total_g_State_dropped = dry_total_g_State.drop('State', axis=1) dry_total_g_State_dropped # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.300162Z","iopub.execute_input":"2025-11-10T02:05:08.300368Z","iopub.status.idle":"2025-11-10T02:05:08.315749Z","shell.execute_reply.started":"2025-11-10T02:05:08.300341Z","shell.execute_reply":"2025-11-10T02:05:08.315052Z"}} # Create custom dataset class StateDataset(Dataset): def __init__(self, df, img_dir, transform=None): self.df = df self.img_dir = img_dir self.transform = transform def __len__(self): return len(self.df) def __getitem__(self, idx): # Get one row from dataframe row = self.df.iloc[idx] # Build full path to image file path = f"{self.img_dir}/{row['image_path']}" # Load image from disk and convert to RGB image = Image.open(path).convert('RGB') # Apply trasnforms (resize to 224x224, normalize, etc) if self.transform: image = self.transform(image) # Get the target value target = torch.tensor(row['state_encoded'], dtype = torch.long) # Get sample_id for reference sample_id = row['sample_id'] # Return tuple: (image_tensor, State, sample_id) return image, target, sample_id train_dataset_state = StateDataset( df=dry_total_g_State, img_dir = '/kaggle/input/dataset/', transform = transform ) train_dataloader_state = DataLoader( dataset=train_dataset_state, shuffle=True, batch_size=32 ) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.316329Z","iopub.execute_input":"2025-11-10T02:05:08.316479Z","iopub.status.idle":"2025-11-10T02:05:08.332650Z","shell.execute_reply.started":"2025-11-10T02:05:08.316467Z","shell.execute_reply":"2025-11-10T02:05:08.332072Z"}} # Making the num classes leS = LabelEncoder() state_encoded = leS.fit_transform(dry_total_g_State['State']) dry_total_g_State['state_encoded'] = leS.fit_transform(dry_total_g_State['State']) state_dict = dry_total_g_State[['State', 'state_encoded']].drop_duplicates().set_index('State')['state_encoded'].to_dict() sorted_state_dict = sorted(state_dict.items()) sorted_state_dict # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.333324Z","iopub.execute_input":"2025-11-10T02:05:08.333565Z","iopub.status.idle":"2025-11-10T02:05:08.346060Z","shell.execute_reply.started":"2025-11-10T02:05:08.333550Z","shell.execute_reply":"2025-11-10T02:05:08.345400Z"}} leS.classes_ # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.346822Z","iopub.execute_input":"2025-11-10T02:05:08.347063Z","iopub.status.idle":"2025-11-10T02:05:08.617407Z","shell.execute_reply.started":"2025-11-10T02:05:08.347043Z","shell.execute_reply":"2025-11-10T02:05:08.616831Z"}} # Creating State model class StateModel(nn.Module): def __init__(self, num_classes): super().__init__() self.model = efficientnet_b0(pretrained=False) self.model.classifier = nn.Sequential( nn.Linear(1280, 128), nn.ReLU(), nn.Linear(128, num_classes) ) def forward(self, x): return self.model(x) num_classes_state = len(leS.classes_) model_state = StateModel(num_classes_state).to(device) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:08.618017Z","iopub.execute_input":"2025-11-10T02:05:08.618240Z","iopub.status.idle":"2025-11-10T02:05:09.451785Z","shell.execute_reply.started":"2025-11-10T02:05:08.618224Z","shell.execute_reply":"2025-11-10T02:05:09.451163Z"}} #Loading the previous models checkpoint = torch.load('/kaggle/input/dataset/all_models.pth', map_location=device) model_height.load_state_dict(checkpoint['model_height']) model_NDVI.load_state_dict(checkpoint['model_NDVI']) model_state.load_state_dict(checkpoint['model_state']) model_species.load_state_dict(checkpoint['model_species']) # %% [markdown] # # Training Step # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.452404Z","iopub.execute_input":"2025-11-10T02:05:09.452580Z","iopub.status.idle":"2025-11-10T02:05:09.456425Z","shell.execute_reply.started":"2025-11-10T02:05:09.452565Z","shell.execute_reply":"2025-11-10T02:05:09.455729Z"}} # # Height model # from tqdm.auto import tqdm # from PIL import Image # torch.manual_seed(42) # epochs = 1 # from torch.optim.lr_scheduler import StepLR # loss_fn = nn.MSELoss() # optimizer = torch.optim.Adam(params=model_height.parameters(), lr = 0.001) # scheduler = StepLR(optimizer, step_size=10, gamma=0.8) # for epoch in tqdm(range(epochs)): # height_loss = 0 # batch_count = 0 # for image, target, sample_id in train_dataloader: # image = image.to(device) # target = target.unsqueeze(1).to(device) # # Forward pass # outputs = model_height(image) # loss = loss_fn(outputs, target) # #Backward pass # optimizer.zero_grad() # loss.backward() # optimizer.step() # height_loss += loss.item() # batch_count += 1 # scheduler.step() # if epoch % 5 == 0: # print(f"Epoch {epoch}, Loss: {height_loss/len(train_dataloader):.6f}\n --------") # .6f instead of .4f # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.457101Z","iopub.execute_input":"2025-11-10T02:05:09.457350Z","iopub.status.idle":"2025-11-10T02:05:09.469900Z","shell.execute_reply.started":"2025-11-10T02:05:09.457334Z","shell.execute_reply":"2025-11-10T02:05:09.469262Z"}} # # NDVI Training # from tqdm.auto import tqdm # from PIL import Image # torch.manual_seed(42) # epochs = 1 # loss_fn = nn.MSELoss() # optimizer = torch.optim.Adam(params=model_NDVI.parameters(), lr = 0.001) # scheduler = StepLR(optimizer, step_size=10, gamma=0.8) # for epoch in tqdm(range(epochs)): # NDVI_loss = 0 # batch_count = 0 # for image, target, sample_id in train_dataloader_NDVI: # image = image.to(device) # target = target.unsqueeze(1).to(device) # # Forward pass # outputs = model_NDVI(image) # loss = loss_fn(outputs, target) # #Backward pass # optimizer.zero_grad() # loss.backward() # optimizer.step() # NDVI_loss += loss.item() # batch_count += 1 # scheduler.step() # if epoch % 5 ==0: # print(f"Epoch {epoch}, Loss: {NDVI_loss/len(train_dataloader):.6f}\n-----") # .6f instead of .4f # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.470518Z","iopub.execute_input":"2025-11-10T02:05:09.470674Z","iopub.status.idle":"2025-11-10T02:05:09.486306Z","shell.execute_reply.started":"2025-11-10T02:05:09.470661Z","shell.execute_reply":"2025-11-10T02:05:09.485603Z"}} # # Species Trainig step # from torch.optim.lr_scheduler import StepLR # torch.manual_seed(42) # epochs = 1 # loss_fn = nn.CrossEntropyLoss() # optimizer = torch.optim.Adam(params=model_species.parameters(), lr = 0.01) # scheduler = StepLR(optimizer, step_size=10, gamma=0.8) # for epoch in tqdm(range(epochs)): # species_loss = 0 # total_samples = 0 # total_correct = 0 # for image, target, sample_id in train_dataloader_species: # image = image.to(device) # target = target.to(device) # #Forward pass # pred = model_species(image) # loss = loss_fn(pred, target) # optimizer.zero_grad() # loss.backward() # optimizer.step() # species_loss += loss.item() # # Calculate accuracy # correct = (pred.argmax(dim=1) == target).sum().item() # total_correct += correct # total_samples += len(target) # scheduler.step() # if epoch % 5 ==0: # species_accuracy = total_correct / total_samples # print(f"Epoch {epoch}, Loss: {species_loss/len(train_dataloader_species):.6f}, Accuracy: {species_accuracy:.2%}\n-------") # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.486895Z","iopub.execute_input":"2025-11-10T02:05:09.487083Z","iopub.status.idle":"2025-11-10T02:05:09.503663Z","shell.execute_reply.started":"2025-11-10T02:05:09.487070Z","shell.execute_reply":"2025-11-10T02:05:09.503104Z"}} # # State Training Step # epochs = 1 # train_loss = 0 # loss_fn = nn.CrossEntropyLoss() # optimizer = torch.optim.Adam(params=model_state.parameters(), lr=0.01) # scheduler = StepLR(optimizer, step_size=10, gamma=0.8) # for epoch in tqdm(range(epochs)): # state_loss = 0 # total_samples = 0 # total_correct = 0 # for image, target, sample_id in train_dataloader_state: # # move to device # image = image.to(device) # target = target.to(device) # # Forward pass # pred = model_state(image) # loss = loss_fn(pred, target) # optimizer.zero_grad() # loss.backward() # optimizer.step() # state_loss += loss.item() # #Calculate accuracy # correct = (pred.argmax(dim=1) == target).sum().item() # total_correct += correct # total_samples += len(target) # scheduler.step() # if epoch % 5 == 0: # state_accuracy = total_correct / total_samples # print(f"Epoch {epoch}, Loss: {state_loss/len(train_dataloader_state):.6f}, Accuracy: {state_accuracy:.2%}\n-----") # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.504323Z","iopub.execute_input":"2025-11-10T02:05:09.504522Z","iopub.status.idle":"2025-11-10T02:05:09.626516Z","shell.execute_reply.started":"2025-11-10T02:05:09.504508Z","shell.execute_reply":"2025-11-10T02:05:09.625977Z"}} # Making the regression model for the dataframe #Pre processing from sklearn.pipeline import Pipeline from sklearn.model_selection import train_test_split from sklearn.preprocessing import OneHotEncoder from sklearn.compose import ColumnTransformer from sklearn.preprocessing import StandardScaler target = train_df['target'] train_notar = train_df.drop(['target','sample_id', 'image_path'], axis=1) cat_cols = train_notar[['State', 'Species', 'target_name']].columns.to_list() num_cols = train_notar.select_dtypes('number').columns.to_list() scaler = StandardScaler() ohe = OneHotEncoder(sparse_output=False, handle_unknown='ignore') num_pipeline = Pipeline(steps=[ ('scaler', scaler) ] ) cat_pipeline = Pipeline(steps=[ ('encoder', ohe) ]) col_transformer = ColumnTransformer(transformers=[ ('num_pipeline', num_pipeline, num_cols), ('cat_pipeline', cat_pipeline, cat_cols) ], remainder="passthrough") col_transformer.set_output(transform='pandas') train_notar_transformed = col_transformer.fit_transform(train_notar) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.627313Z","iopub.execute_input":"2025-11-10T02:05:09.627525Z","iopub.status.idle":"2025-11-10T02:05:09.631662Z","shell.execute_reply.started":"2025-11-10T02:05:09.627509Z","shell.execute_reply":"2025-11-10T02:05:09.630784Z"}} train_notar = train_notar.drop('Sampling_Date', axis=1) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.632418Z","iopub.execute_input":"2025-11-10T02:05:09.632696Z","iopub.status.idle":"2025-11-10T02:05:09.675715Z","shell.execute_reply.started":"2025-11-10T02:05:09.632673Z","shell.execute_reply":"2025-11-10T02:05:09.674970Z"}} train_notar_transformed = col_transformer.fit_transform(train_notar) train_notar_transformed # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.676563Z","iopub.execute_input":"2025-11-10T02:05:09.676796Z","iopub.status.idle":"2025-11-10T02:05:09.873947Z","shell.execute_reply.started":"2025-11-10T02:05:09.676771Z","shell.execute_reply":"2025-11-10T02:05:09.873185Z"}} # Saving my state_dicrs torch.save({ 'model_height': model_height.state_dict(), 'model_NDVI': model_NDVI.state_dict(), 'model_state': model_state.state_dict(), 'model_species': model_species.state_dict() }, 'all_models.pth') # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:09.874922Z","iopub.execute_input":"2025-11-10T02:05:09.875190Z","iopub.status.idle":"2025-11-10T02:05:10.338104Z","shell.execute_reply.started":"2025-11-10T02:05:09.875171Z","shell.execute_reply":"2025-11-10T02:05:10.337410Z"}} # make the regression model from sklearn.ensemble import HistGradientBoostingRegressor from sklearn.linear_model import LinearRegression from sklearn.model_selection import train_test_split import xgboost as xgb X = train_notar_transformed y = target X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:10.338993Z","iopub.execute_input":"2025-11-10T02:05:10.339561Z","iopub.status.idle":"2025-11-10T02:05:10.343545Z","shell.execute_reply.started":"2025-11-10T02:05:10.339539Z","shell.execute_reply":"2025-11-10T02:05:10.342775Z"}} print(f"Shape: {X.shape}") # (1785, ?) print(f"Features: {X.shape[1]}") # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:10.344289Z","iopub.execute_input":"2025-11-10T02:05:10.344558Z","iopub.status.idle":"2025-11-10T02:05:10.513557Z","shell.execute_reply.started":"2025-11-10T02:05:10.344535Z","shell.execute_reply":"2025-11-10T02:05:10.512926Z"}} # GradientBoosting (older, more stable than Hist version) from sklearn.ensemble import GradientBoostingRegressor model_gbr = GradientBoostingRegressor( n_estimators=100, learning_rate=0.1, max_depth=5, random_state=42 ) # Transform training data train_notar_transformed = col_transformer.fit_transform(train_notar) # Train GBR (it's fast, just keep this) from sklearn.ensemble import GradientBoostingRegressor model_gbr = GradientBoostingRegressor(n_estimators=100, random_state=42) model_gbr.fit(train_notar_transformed, target) print(f"✓ GBR trained") # Transform test data and predict train_gbr_score = model_gbr.score(X_train, y_train) test_gbr_score = model_gbr.score(X_test, y_test) print(f"Train R²: {train_gbr_score:.4f}") print(f"Test R²: {test_gbr_score:.4f}") # %% [code] # %% [code] # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:10.514291Z","iopub.execute_input":"2025-11-10T02:05:10.514531Z","iopub.status.idle":"2025-11-10T02:05:11.414660Z","shell.execute_reply.started":"2025-11-10T02:05:10.514504Z","shell.execute_reply":"2025-11-10T02:05:11.414118Z"}} # Get unique images unique_images = test_df['image_path'].unique() # Create lists to store predictions all_states = [] all_species = [] all_ndvi = [] all_heights = [] # Predict for each unique image for img_path in unique_images: # Load and process image path = '/kaggle/input/csiro-biomass/' + img_path image = Image.open(path).convert('RGB') image = transform(image).unsqueeze(0).to(device) # Predict features with torch.no_grad(): ndvi_pred = model_NDVI(image).detach().cpu().numpy().item() height_pred = model_height(image).detach().cpu().numpy().item() species_pred = model_species(image).argmax(dim=1).item() species = le.inverse_transform([species_pred])[0] state_pred = model_state(image).argmax(dim=1).item() state = leS.inverse_transform([state_pred])[0] # Get number of rows for this image (should be 5 - one per target_name) num_rows = len(test_df[test_df['image_path'] == img_path]) # Append predictions (repeated for each target_name) all_states.extend([state] * num_rows) all_species.extend([species] * num_rows) all_ndvi.extend([ndvi_pred] * num_rows) all_heights.extend([height_pred] * num_rows) # Assign to test_df test_df['State'] = all_states test_df['Species'] = all_species test_df['Pre_GSHH_NDVI'] = all_ndvi test_df['Height_Ave_cm'] = all_heights # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.415302Z","iopub.execute_input":"2025-11-10T02:05:11.415554Z","iopub.status.idle":"2025-11-10T02:05:11.419947Z","shell.execute_reply.started":"2025-11-10T02:05:11.415532Z","shell.execute_reply":"2025-11-10T02:05:11.419160Z"}} print(type(model_gbr)) # Should be GradientBoostingRegressor, not ndarray # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.420762Z","iopub.execute_input":"2025-11-10T02:05:11.421021Z","iopub.status.idle":"2025-11-10T02:05:11.441126Z","shell.execute_reply.started":"2025-11-10T02:05:11.420995Z","shell.execute_reply":"2025-11-10T02:05:11.440594Z"}} # Transform test data using the same pipeline test_df_nopath = test_df.drop(['sample_id', 'image_path'], axis=1) test_df_transformed = col_transformer.transform(test_df_nopath) # Predict with GBR actual_preds = model_gbr.predict(test_df_transformed) # Create submission submission_df = test_df[['sample_id']].copy() submission_df['target'] = actual_preds # Save submission_df.to_csv('submission.csv', index=False) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.441852Z","iopub.execute_input":"2025-11-10T02:05:11.442076Z","iopub.status.idle":"2025-11-10T02:05:11.457464Z","shell.execute_reply.started":"2025-11-10T02:05:11.442060Z","shell.execute_reply":"2025-11-10T02:05:11.456831Z"}} submission_df # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.458171Z","iopub.execute_input":"2025-11-10T02:05:11.458490Z","iopub.status.idle":"2025-11-10T02:05:11.480543Z","shell.execute_reply.started":"2025-11-10T02:05:11.458474Z","shell.execute_reply":"2025-11-10T02:05:11.479869Z"}} print("="*50) print("FINAL VALIDATION BEFORE SUBMISSION") print("="*50) print(f"\n1. Test data:") print(f" Rows: {len(test_df)}") print(f" Unique images: {test_df['image_path'].nunique()}") print(f" Sample IDs: {len(test_df['sample_id'].unique())}") print(f"\n2. Predictions:") print(f" Length: {len(actual_preds)}") print(f" Type: {type(actual_preds)}") print(f" Min: {actual_preds.min():.3f}") print(f" Max: {actual_preds.max():.3f}") print(f" Mean: {actual_preds.mean():.3f}") print(f" NaN count: {pd.isna(actual_preds).sum()}") print(f" Inf count: {np.isinf(actual_preds).sum()}") print(f"\n3. Submission:") print(f" Shape: {submission_df.shape}") print(f" Columns: {submission_df.columns.tolist()}") print(f" Dtypes:\n{submission_df.dtypes}") print(f"\n4. Sample submission comparison:") sample_sub = pd.read_csv('/kaggle/input/csiro-biomass/sample_submission.csv') print(f" Expected rows: {len(sample_sub)}") print(f" Actual rows: {len(submission_df)}") print(f" Expected columns: {sample_sub.columns.tolist()}") print(f" Actual columns: {submission_df.columns.tolist()}") print(f"\n5. First 10 rows of submission:") print(submission_df.head(10)) print(f"\n6. Last 10 rows of submission:") print(submission_df.tail(10)) # Assertions assert len(submission_df) == len(test_df), f"❌ Row mismatch: {len(submission_df)} vs {len(test_df)}" assert len(submission_df) == len(sample_sub), f"❌ Doesn't match sample_submission: {len(submission_df)} vs {len(sample_sub)}" assert submission_df.columns.tolist() == ['sample_id', 'target'], f"❌ Wrong columns" assert submission_df.isnull().sum().sum() == 0, "❌ Has null values" assert np.isfinite(submission_df['target']).all(), "❌ Has inf values" print("\n" + "="*50) print("✓ ALL CHECKS PASSED") print("="*50) submission_df.to_csv('submission.csv', index=False) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.481189Z","iopub.execute_input":"2025-11-10T02:05:11.481442Z","iopub.status.idle":"2025-11-10T02:05:11.491089Z","shell.execute_reply.started":"2025-11-10T02:05:11.481426Z","shell.execute_reply":"2025-11-10T02:05:11.490528Z"}} # 1. Ensure target is float submission_df['target'] = submission_df['target'].astype('float64') # 2. Strip whitespace from column names submission_df.columns = submission_df.columns.str.strip() # 3. Strip whitespace from sample_id values submission_df['sample_id'] = submission_df['sample_id'].str.strip() # 4. Verify print("Column names:", submission_df.columns.tolist()) print("Target dtype:", submission_df['target'].dtype) print("Sample IDs (first 3):", submission_df['sample_id'].head(3).tolist()) # 5. Final check - match sample_submission exactly sample_sub = pd.read_csv('/kaggle/input/csiro-biomass/sample_submission.csv') assert submission_df['sample_id'].tolist() == sample_sub['sample_id'].tolist(), "Sample IDs don't match order" print("✓ Sample IDs match sample_submission exactly") # Save submission_df.to_csv('submission.csv', index=False) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.491833Z","iopub.execute_input":"2025-11-10T02:05:11.492022Z","iopub.status.idle":"2025-11-10T02:05:11.511181Z","shell.execute_reply.started":"2025-11-10T02:05:11.492008Z","shell.execute_reply":"2025-11-10T02:05:11.510575Z"}} import pandas as pd # Load test data test_df = pd.read_csv('/kaggle/input/csiro-biomass/test.csv') # Create submission with dummy predictions submission_df = test_df[['sample_id']].copy() submission_df['target'] = 0.0 # Dummy predictions # Save submission_df.to_csv('submission.csv', index=False) print(f"Created submission with {len(submission_df)} rows") print(submission_df.head()) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.511950Z","iopub.execute_input":"2025-11-10T02:05:11.512275Z","iopub.status.idle":"2025-11-10T02:05:11.528473Z","shell.execute_reply.started":"2025-11-10T02:05:11.512235Z","shell.execute_reply":"2025-11-10T02:05:11.527727Z"}} import pandas as pd import os print("="*50) print("DEBUGGING SUBMISSION CREATION") print("="*50) # Check test.csv test_df = pd.read_csv('/kaggle/input/csiro-biomass/test.csv') print(f"\n1. Test data loaded:") print(f" Shape: {test_df.shape}") print(f" Columns: {test_df.columns.tolist()}") print(f" First 5 rows:\n{test_df.head()}") # Create submission submission_df = test_df[['sample_id']].copy() submission_df['target'] = 0.0 print(f"\n2. Submission created:") print(f" Shape: {submission_df.shape}") print(f" Columns: {submission_df.columns.tolist()}") print(f" Dtypes:\n{submission_df.dtypes}") print(f" First 5 rows:\n{submission_df.head()}") print(f" Last 5 rows:\n{submission_df.tail()}") # Check for issues print(f"\n3. Data quality:") print(f" Null values: {submission_df.isnull().sum().sum()}") print(f" Duplicate sample_ids: {submission_df['sample_id'].duplicated().sum()}") print(f" Target dtype: {submission_df['target'].dtype}") print(f" Target range: {submission_df['target'].min()} to {submission_df['target'].max()}") # Save submission_df.to_csv('submission.csv', index=False) print("\n4. Saved to submission.csv") # Verify saved file if os.path.exists('submission.csv'): verify = pd.read_csv('submission.csv') print(f" Verified: {len(verify)} rows saved") print(f" First row: {verify.iloc[0].to_dict()}") else: print(" ERROR: submission.csv not created!") print("="*50) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.529192Z","iopub.execute_input":"2025-11-10T02:05:11.529532Z","iopub.status.idle":"2025-11-10T02:05:11.538917Z","shell.execute_reply.started":"2025-11-10T02:05:11.529512Z","shell.execute_reply":"2025-11-10T02:05:11.538279Z"}} import pandas as pd import numpy as np test_df = pd.read_csv('/kaggle/input/csiro-biomass/test.csv') submission_df = test_df[['sample_id']].copy() # Random values between 5 and 50 (reasonable biomass range) np.random.seed(42) submission_df['target'] = np.random.uniform(5, 50, size=len(test_df)) submission_df.to_csv('submission.csv', index=False) print(f"Created submission with {len(submission_df)} rows") print(submission_df.head()) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:05:11.539707Z","iopub.execute_input":"2025-11-10T02:05:11.540267Z","iopub.status.idle":"2025-11-10T02:05:11.553688Z","shell.execute_reply.started":"2025-11-10T02:05:11.540234Z","shell.execute_reply":"2025-11-10T02:05:11.553075Z"}} # After getting predictions rows = [] for idx, row in test_df.iterrows(): sample_id = row['sample_id'] target_value = actual_preds[idx] # Key: ensure non-negative and explicit float rows.append([sample_id, max(0.0, float(target_value))]) # Create submission from rows submission_df = pd.DataFrame(rows, columns=["sample_id", "target"]) # Save submission_df.to_csv('submission.csv', index=False) print(f"Submission saved: {len(submission_df)} rows") print(submission_df.head(10)) # %% [code] {"execution":{"iopub.status.busy":"2025-11-10T02:06:49.245157Z","iopub.execute_input":"2025-11-10T02:06:49.245770Z","iopub.status.idle":"2025-11-10T02:06:49.252394Z","shell.execute_reply.started":"2025-11-10T02:06:49.245742Z","shell.execute_reply":"2025-11-10T02:06:49.251661Z"}} submission_df.dtypes # %% [code]