{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":64733,"databundleVersionId":8171035,"sourceType":"competition"}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:17:15.941709Z","iopub.execute_input":"2024-07-09T07:17:15.942192Z","iopub.status.idle":"2024-07-09T07:17:16.347868Z","shell.execute_reply.started":"2024-07-09T07:17:15.942131Z","shell.execute_reply":"2024-07-09T07:17:16.346801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define n iterations\nn = 5\n\n# load metadata\n\nmetadata = pd.read_csv('/kaggle/input/geolifeclef-2024/GLC24_PA_metadata_train.csv')\n\n# get survey ids\nsurvey_ids = metadata['surveyId'].unique()\nsurvey_ids\n\n# define train ratio\ntrain_ratio = 0.8\n\n# shuffle survey ids\n\n\ndef get_train_val_survey_ids(survey_ids, train_ratio):\n    \n    # shuffle survey ids\n    np.random.shuffle(survey_ids)\n    # split survey ids into train and val\n    n_train = int(train_ratio * len(survey_ids))\n    train_survey_ids = survey_ids[:n_train]\n    val_survey_ids = survey_ids[n_train:]\n    return train_survey_ids, val_survey_ids\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:17:16.349482Z","iopub.execute_input":"2024-07-09T07:17:16.351963Z","iopub.status.idle":"2024-07-09T07:17:18.024857Z","shell.execute_reply.started":"2024-07-09T07:17:16.351935Z","shell.execute_reply":"2024-07-09T07:17:18.024066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 11255 # Number of all unique classes within the PO and PA data.","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:17:18.026080Z","iopub.execute_input":"2024-07-09T07:17:18.026545Z","iopub.status.idle":"2024-07-09T07:17:18.031127Z","shell.execute_reply.started":"2024-07-09T07:17:18.026505Z","shell.execute_reply":"2024-07-09T07:17:18.030203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport tqdm\nimport numpy as np\nimport pandas as pd\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom sklearn.metrics import precision_recall_fscore_support","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:17:18.032173Z","iopub.execute_input":"2024-07-09T07:17:18.032425Z","iopub.status.idle":"2024-07-09T07:17:21.181400Z","shell.execute_reply.started":"2024-07-09T07:17:18.032402Z","shell.execute_reply":"2024-07-09T07:17:21.180500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\n\ndef construct_patch_path(data_path, survey_id):\n    \"\"\"Construct the patch file path based on plot_id as './CD/AB/XXXXABCD.jpeg'\"\"\"\n    path = data_path\n    for d in (str(survey_id)[-2:], str(survey_id)[-4:-2]):\n        path = os.path.join(path, d)\n\n    path = os.path.join(path, f\"{survey_id}.jpeg\")\n\n    return path\n\nclass BaselineTrainDataset(Dataset):\n    def __init__(self, bioclim_data_dir, landsat_data_dir, sentinel_data_dir, metadata, survey_ids, transform=None):\n        self.transform = transform\n        self.sentinel_transform = transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize(mean=(0.5, 0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5, 0.5)),\n        ])\n      \n        self.bioclim_data_dir = bioclim_data_dir\n        self.landsat_data_dir = landsat_data_dir\n        self.sentinel_data_dir = sentinel_data_dir\n        self.metadata = metadata\n        self.metadata = self.metadata.dropna(subset=\"speciesId\").reset_index(drop=True)\n        self.metadata['speciesId'] = self.metadata['speciesId'].astype(int)\n        self.label_dict = self.metadata.groupby('surveyId')['speciesId'].apply(list).to_dict()\n        \n        self.metadata = self.metadata.drop_duplicates(subset=\"surveyId\").reset_index(drop=True)\n\n        self.survey_ids = survey_ids\n\n    def __len__(self):\n        return len(self.survey_ids)\n\n    def __getitem__(self, idx):\n        \n        survey_id = self.survey_ids[idx]\n        \n        landsat_sample = torch.nan_to_num(torch.load(os.path.join(self.landsat_data_dir, f\"GLC24-PA-train-landsat-time-series_{survey_id}_cube.pt\")))\n        bioclim_sample = torch.nan_to_num(torch.load(os.path.join(self.bioclim_data_dir, f\"GLC24-PA-train-bioclimatic_monthly_{survey_id}_cube.pt\")))\n\n        rgb_sample = np.array(Image.open(construct_patch_path(self.sentinel_data_dir, survey_id)))\n        nir_sample = np.array(Image.open(construct_patch_path(self.sentinel_data_dir.replace(\"rgb\", \"nir\").replace(\"RGB\", \"NIR\"), survey_id)))\n        sentinel_sample = np.concatenate((rgb_sample, nir_sample[...,None]), axis=2)\n\n        species_ids = self.label_dict.get(survey_id, [])  # Get list of species IDs for the survey ID\n        label = torch.zeros(num_classes)  # Initialize label tensor\n        for species_id in species_ids:\n            label_id = species_id\n            label[label_id] = 1  # Set the corresponding class index to 1 for each species\n        \n        if isinstance(landsat_sample, torch.Tensor):\n            landsat_sample = landsat_sample.permute(1, 2, 0)  # Change tensor shape from (C, H, W) to (H, W, C)\n            landsat_sample = landsat_sample.numpy()  # Convert tensor to numpy array\n            \n        if isinstance(bioclim_sample, torch.Tensor):\n            bioclim_sample = bioclim_sample.permute(1, 2, 0)  # Change tensor shape from (C, H, W) to (H, W, C)\n            bioclim_sample = bioclim_sample.numpy()  # Convert tensor to numpy array   \n        \n        if self.transform:\n            landsat_sample = self.transform(landsat_sample)\n            bioclim_sample = self.transform(bioclim_sample)\n            sentinel_sample = self.sentinel_transform(sentinel_sample)\n\n        return landsat_sample, bioclim_sample, sentinel_sample, label, survey_id\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:17:21.184653Z","iopub.execute_input":"2024-07-09T07:17:21.185512Z","iopub.status.idle":"2024-07-09T07:17:21.202972Z","shell.execute_reply.started":"2024-07-09T07:17:21.185469Z","shell.execute_reply":"2024-07-09T07:17:21.201867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Dataset and DataLoader\n\ndef create_loaders(train_survey_ids, val_survey_ids):\n  train_batch_size = 64\n  val_batch_size = 64\n  transform = transforms.Compose([\n      transforms.ToTensor()\n  ])\n\n  # Load Training metadata\n  train_landsat_data_path = \"/kaggle/input/geolifeclef-2024/TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-landsat_time_series\"\n  train_bioclim_data_path = \"/kaggle/input/geolifeclef-2024/TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-bioclimatic_monthly\"\n  train_sentinel_data_path=\"/kaggle/input/geolifeclef-2024/PA_Train_SatellitePatches_RGB/pa_train_patches_rgb\"\n  train_metadata_path = \"/kaggle/input/geolifeclef-2024/GLC24_PA_metadata_train.csv\"\n\n  train_metadata = pd.read_csv(train_metadata_path)\n\n  train_data = BaselineTrainDataset(train_bioclim_data_path, train_landsat_data_path, train_sentinel_data_path, train_metadata, train_survey_ids, transform=transform)\n  train_loader = DataLoader(train_data, batch_size=train_batch_size, shuffle=False, num_workers=4)\n\n  val_data = BaselineTrainDataset(train_bioclim_data_path, train_landsat_data_path, train_sentinel_data_path, train_metadata, val_survey_ids, transform=transform)\n  val_loader = DataLoader(val_data, batch_size=val_batch_size, shuffle=False, num_workers=4)\n\n  return train_loader, val_loader\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:17:21.204292Z","iopub.execute_input":"2024-07-09T07:17:21.204720Z","iopub.status.idle":"2024-07-09T07:17:21.217114Z","shell.execute_reply.started":"2024-07-09T07:17:21.204689Z","shell.execute_reply":"2024-07-09T07:17:21.216193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom tqdm import tqdm\n\ndef calculate_f1_score_from_tensors(y_true, y_pred, threshold=0.5):\n    \n    y_pred = (y_pred >= threshold)\n    y_true = y_true.cpu().bool()\n    y_pred = y_pred.cpu().bool()\n    \n    TP = (y_true & y_pred).sum(axis=1)  # True Positives per sample\n    FP = (y_true & ~y_pred).sum(axis=1)  # False Positives per sample\n    FN = (~y_true & y_pred).sum(axis=1)  # False Negatives per sample\n\n    # compute f1 score for each sample\n    pre = TP/(TP+FP)\n    rec = TP/(TP+FN)\n    f1 = 2 * pre * rec / (pre + rec)\n\n    # Handle division by zero\n    f1 = np.nan_to_num(f1)\n\n    # compute micro-average f1 score\n    micro_f1 = np.mean(f1)\n    # Return mean F1 score across all samples\n    return micro_f1\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:17:21.218330Z","iopub.execute_input":"2024-07-09T07:17:21.218744Z","iopub.status.idle":"2024-07-09T07:17:21.229918Z","shell.execute_reply.started":"2024-07-09T07:17:21.218711Z","shell.execute_reply":"2024-07-09T07:17:21.229059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if cuda is available\ndevice = torch.device(\"mps\")\n\nif torch.cuda.is_available():\n    device = torch.device(\"cuda\")\n    \n\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:17:21.231732Z","iopub.execute_input":"2024-07-09T07:17:21.232359Z","iopub.status.idle":"2024-07-09T07:17:21.280634Z","shell.execute_reply.started":"2024-07-09T07:17:21.232327Z","shell.execute_reply":"2024-07-09T07:17:21.279703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F\n\nclass MultimodalEnsemble(nn.Module):\n    def __init__(self, num_classes):\n        super(MultimodalEnsemble, self).__init__()\n        \n        self.landsat_norm = nn.LayerNorm([6,4,21])\n        self.landsat_model = models.resnet18(weights=None)\n        # Modify the first convolutional layer to accept 6 channels instead of 3\n        self.landsat_model.conv1 = nn.Conv2d(6, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.landsat_model.maxpool = nn.Identity()\n        self.ls_ln = nn.LayerNorm(1000)\n        self.ls_fc1 = nn.Linear(1000, 4096)\n        self.ls_dropout = nn.Dropout(p=0.1)\n        self.ls_fc2 = nn.Linear(4096, num_classes)\n        \n        self.bioclim_norm = nn.LayerNorm([4,19,12])\n        self.bioclim_model = models.resnet18(weights=None)  \n        # Modify the first convolutional layer to accept 4 channels instead of 3\n        self.bioclim_model.conv1 = nn.Conv2d(4, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bioclim_model.maxpool = nn.Identity()\n        self.bc_ln = nn.LayerNorm(1000)\n        self.bc_fc1 = nn.Linear(1000, 4096)\n        self.bc_dropout = nn.Dropout(p=0.1)\n        self.bc_fc2 = nn.Linear(4096, num_classes)\n        \n        self.sentinel_model = models.swin_t(weights=\"IMAGENET1K_V1\")\n        # Modify the first layer to accept 4 channels instead of 3\n        self.sentinel_model.features[0][0] = nn.Conv2d(4, 96, kernel_size=(4, 4), stride=(4, 4))\n        self.sentinel_model.head = nn.Identity()\n        self.se_ln = nn.LayerNorm(768)\n        self.se_fc1 = nn.Linear(768, 4096)\n        self.se_dropout = nn.Dropout(p=0.1)\n        self.se_fc2 = nn.Linear(4096, num_classes)\n        \n        \n    def forward(self, x, y, z):\n        \n        x = self.landsat_norm(x)\n        x = self.landsat_model(x)\n        x = self.bc_ln(x)\n        x = self.ls_fc1(x)\n        x = self.ls_dropout(x)\n        x = self.ls_fc2(x)\n        \n        y = self.bioclim_norm(y)\n        y = self.bioclim_model(y)\n        y = self.bc_ln(y)\n        y = self.bc_fc1(y)\n        y = self.bc_dropout(y)\n        y = self.bc_fc2(y)\n        \n        z = self.sentinel_model(z)\n        z = self.se_ln(z)\n        z = self.se_fc1(z)\n        z = self.se_dropout(z)\n        z = self.se_fc2(z)\n        \n        # average the predictions\n        out = (x + y + z) / 3\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:20:47.273561Z","iopub.execute_input":"2024-07-09T07:20:47.274243Z","iopub.status.idle":"2024-07-09T07:20:47.289319Z","shell.execute_reply.started":"2024-07-09T07:20:47.274209Z","shell.execute_reply":"2024-07-09T07:20:47.288425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop with custom F1 score\nf1s_fme = []\n\nfor i in range(n):\n\n    # get train and val survey ids\n    train_survey_ids, val_survey_ids = get_train_val_survey_ids(survey_ids, train_ratio)\n\n    train_loader, val_loader = create_loaders(train_survey_ids, val_survey_ids)\n    \n    model = MultimodalEnsemble(num_classes=num_classes)\n    model.to(device)\n\n    # Hyperparameters\n    learning_rate = 0.00025\n    num_epochs = 10\n    positive_weigh_factor = 1.0\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n    scheduler = CosineAnnealingLR(optimizer, T_max=25, verbose=True)\n\n\n    # Training loop\n    for epoch in range(num_epochs):\n        model.train()\n        \n        for data1, data2, data3, targets, _ in train_loader:\n\n            data1 = data1.to(device)\n            data2 = data2.to(device)\n            data3 = data3.to(device)\n            targets = targets.to(device)\n\n            optimizer.zero_grad()\n            outputs = model(data1, data2, data3)\n\n            pos_weight = targets*positive_weigh_factor  # All positive weights are equal to 10\n            criterion = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n            loss = criterion(outputs, targets)\n\n            loss.backward()\n            optimizer.step()\n        \n        print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.4f}')\n\n    print(\"Training complete.\")\n\n    # Predict with trained models on validation set\n    model.eval()\n    f1s_iteration = []\n\n    for data1, data2, data3, targets, _ in tqdm(val_loader):\n        data1 = data1.to(device)\n        data2 = data2.to(device)\n        data3 = data3.to(device)\n        targets = targets.to(device)\n\n        with torch.no_grad():\n            outputs = model(data1, data2, data3)\n            outputs = torch.sigmoid(outputs)\n\n        # Calculate custom F1 score\n        f1 = calculate_f1_score_from_tensors(targets, outputs, threshold=0.5)\n        f1s_iteration.append(f1)\n\n    # Calculate mean F1 score for the iteration\n    mean_f1_iteration = np.mean(f1s_iteration)\n    print(f\"Iteration {i+1} - Mean F1 Score: {mean_f1_iteration}\")\n\n    # Append mean F1 score to list\n    f1s_fme.append(mean_f1_iteration)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:20:47.538513Z","iopub.execute_input":"2024-07-09T07:20:47.539313Z","iopub.status.idle":"2024-07-09T07:31:40.506108Z","shell.execute_reply.started":"2024-07-09T07:20:47.539282Z","shell.execute_reply":"2024-07-09T07:31:40.504506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\n\nnow = datetime.datetime.now()\ntimestamp = now.strftime('%Y-%m-%d_%H-%M-%S')\n\nresults = pd.DataFrame(f1s_fme, columns=['fme_f1'])\nresults.to_csv(f'{timestamp}_fme_benchmark_results_{n}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T07:31:40.507040Z","iopub.status.idle":"2024-07-09T07:31:40.507379Z","shell.execute_reply.started":"2024-07-09T07:31:40.507221Z","shell.execute_reply":"2024-07-09T07:31:40.507234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- code should work but needs to be run in kaggle or colab for gpu. ","metadata":{}}]}