{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":64733,"databundleVersionId":8171035,"sourceType":"competition"}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom sklearn.neighbors import NearestNeighbors\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T09:19:09.089926Z","iopub.execute_input":"2024-07-09T09:19:09.090312Z","iopub.status.idle":"2024-07-09T09:19:09.095641Z","shell.execute_reply.started":"2024-07-09T09:19:09.090282Z","shell.execute_reply":"2024-07-09T09:19:09.094527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define n iterations\nn = 5\n\n# load metadata\n\nmetadata = pd.read_csv('/kaggle/input/geolifeclef-2024/GLC24_PA_metadata_train.csv')\n\n# get survey ids\nsurvey_ids = metadata['surveyId'].unique()\nsurvey_ids\n\n# define train ratio\ntrain_ratio = 0.8\n\n# shuffle survey ids\n\n\ndef get_train_val_survey_ids(survey_ids, train_ratio):\n    \n    # shuffle survey ids\n    np.random.shuffle(survey_ids)\n    # split survey ids into train and val\n    n_train = int(train_ratio * len(survey_ids))\n    train_survey_ids = survey_ids[:n_train]\n    val_survey_ids = survey_ids[n_train:]\n    return train_survey_ids, val_survey_ids\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T09:19:09.097413Z","iopub.execute_input":"2024-07-09T09:19:09.097754Z","iopub.status.idle":"2024-07-09T09:19:10.789886Z","shell.execute_reply.started":"2024-07-09T09:19:09.097726Z","shell.execute_reply":"2024-07-09T09:19:10.788846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 11255 # Number of all unique classes within the PO and PA data.","metadata":{"execution":{"iopub.status.busy":"2024-07-09T09:19:10.791137Z","iopub.execute_input":"2024-07-09T09:19:10.791444Z","iopub.status.idle":"2024-07-09T09:19:10.795670Z","shell.execute_reply.started":"2024-07-09T09:19:10.791418Z","shell.execute_reply":"2024-07-09T09:19:10.794748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport tqdm\nimport numpy as np\nimport pandas as pd\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom sklearn.metrics import precision_recall_fscore_support","metadata":{"execution":{"iopub.status.busy":"2024-07-09T09:19:10.796880Z","iopub.execute_input":"2024-07-09T09:19:10.797158Z","iopub.status.idle":"2024-07-09T09:19:10.808036Z","shell.execute_reply.started":"2024-07-09T09:19:10.797134Z","shell.execute_reply":"2024-07-09T09:19:10.807220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\n\ndef construct_patch_path(data_path, survey_id):\n    \"\"\"Construct the patch file path based on plot_id as './CD/AB/XXXXABCD.jpeg'\"\"\"\n    path = data_path\n    for d in (str(survey_id)[-2:], str(survey_id)[-4:-2]):\n        path = os.path.join(path, d)\n\n    path = os.path.join(path, f\"{survey_id}.jpeg\")\n\n    return path\n\nclass BaselineTrainDataset(Dataset):\n    def __init__(self, bioclim_data_dir, landsat_data_dir, sentinel_data_dir, metadata, survey_ids, transform=None):\n        self.transform = transform\n        self.sentinel_transform = transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize(mean=(0.5, 0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5, 0.5)),\n        ])\n      \n        self.bioclim_data_dir = bioclim_data_dir\n        self.landsat_data_dir = landsat_data_dir\n        self.sentinel_data_dir = sentinel_data_dir\n        self.metadata = metadata\n        self.metadata = self.metadata.dropna(subset=\"speciesId\").reset_index(drop=True)\n        self.metadata['speciesId'] = self.metadata['speciesId'].astype(int)\n        self.label_dict = self.metadata.groupby('surveyId')['speciesId'].apply(list).to_dict()\n        \n        self.metadata = self.metadata.drop_duplicates(subset=\"surveyId\").reset_index(drop=True)\n\n        self.survey_ids = survey_ids\n\n    def __len__(self):\n        return len(self.survey_ids)\n\n    def __getitem__(self, idx):\n        \n        survey_id = self.survey_ids[idx]\n        \n        landsat_sample = torch.nan_to_num(torch.load(os.path.join(self.landsat_data_dir, f\"GLC24-PA-train-landsat-time-series_{survey_id}_cube.pt\")))\n        bioclim_sample = torch.nan_to_num(torch.load(os.path.join(self.bioclim_data_dir, f\"GLC24-PA-train-bioclimatic_monthly_{survey_id}_cube.pt\")))\n\n        rgb_sample = np.array(Image.open(construct_patch_path(self.sentinel_data_dir, survey_id)))\n        nir_sample = np.array(Image.open(construct_patch_path(self.sentinel_data_dir.replace(\"rgb\", \"nir\").replace(\"RGB\", \"NIR\"), survey_id)))\n        sentinel_sample = np.concatenate((rgb_sample, nir_sample[...,None]), axis=2)\n\n        species_ids = self.label_dict.get(survey_id, [])  # Get list of species IDs for the survey ID\n        label = torch.zeros(num_classes)  # Initialize label tensor\n        for species_id in species_ids:\n            label_id = species_id\n            label[label_id] = 1  # Set the corresponding class index to 1 for each species\n        \n        if isinstance(landsat_sample, torch.Tensor):\n            landsat_sample = landsat_sample.permute(1, 2, 0)  # Change tensor shape from (C, H, W) to (H, W, C)\n            landsat_sample = landsat_sample.numpy()  # Convert tensor to numpy array\n            \n        if isinstance(bioclim_sample, torch.Tensor):\n            bioclim_sample = bioclim_sample.permute(1, 2, 0)  # Change tensor shape from (C, H, W) to (H, W, C)\n            bioclim_sample = bioclim_sample.numpy()  # Convert tensor to numpy array   \n        \n        if self.transform:\n            landsat_sample = self.transform(landsat_sample)\n            bioclim_sample = self.transform(bioclim_sample)\n            sentinel_sample = self.sentinel_transform(sentinel_sample)\n\n        return landsat_sample, bioclim_sample, sentinel_sample, label, survey_id\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T09:19:10.810423Z","iopub.execute_input":"2024-07-09T09:19:10.810770Z","iopub.status.idle":"2024-07-09T09:19:10.829116Z","shell.execute_reply.started":"2024-07-09T09:19:10.810744Z","shell.execute_reply":"2024-07-09T09:19:10.828109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Dataset and DataLoader\n\ndef create_loaders(train_survey_ids, val_survey_ids):\n  train_batch_size = 256\n  val_batch_size = 64\n  transform = transforms.Compose([\n      transforms.ToTensor()\n  ])\n\n   # Load Training metadata\n  train_landsat_data_path = \"/kaggle/input/geolifeclef-2024/TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-landsat_time_series\"\n  train_bioclim_data_path = \"/kaggle/input/geolifeclef-2024/TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-bioclimatic_monthly\"\n  train_sentinel_data_path=\"/kaggle/input/geolifeclef-2024/PA_Train_SatellitePatches_RGB/pa_train_patches_rgb\"\n  train_metadata_path = \"/kaggle/input/geolifeclef-2024/GLC24_PA_metadata_train.csv\"\n\n  train_metadata = pd.read_csv(train_metadata_path)\n\n  train_data = BaselineTrainDataset(train_bioclim_data_path, train_landsat_data_path, train_sentinel_data_path, train_metadata, train_survey_ids, transform=transform)\n  train_loader = DataLoader(train_data, batch_size=train_batch_size, shuffle=False, num_workers=4)\n\n  val_data = BaselineTrainDataset(train_bioclim_data_path, train_landsat_data_path, train_sentinel_data_path, train_metadata, val_survey_ids, transform=transform)\n  val_loader = DataLoader(val_data, batch_size=val_batch_size, shuffle=False, num_workers=4)\n\n  return train_loader, val_loader\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T09:19:10.830349Z","iopub.execute_input":"2024-07-09T09:19:10.830645Z","iopub.status.idle":"2024-07-09T09:19:10.844014Z","shell.execute_reply.started":"2024-07-09T09:19:10.830620Z","shell.execute_reply":"2024-07-09T09:19:10.843025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom tqdm import tqdm\n\ndef calculate_f1_score_from_tensors(y_true, y_pred, threshold=0.5):\n    \n    y_pred = (y_pred >= threshold)\n    \n    TP = (y_true & y_pred).sum(axis=1)  # True Positives per sample\n    FP = (y_true & ~y_pred).sum(axis=1)  # False Positives per sample\n    FN = (~y_true & y_pred).sum(axis=1)  # False Negatives per sample\n\n    # compute f1 score for each sample\n    pre = TP/(TP+FP)\n    rec = TP/(TP+FN)\n    f1 = 2 * pre * rec / (pre + rec)\n\n    # Handle division by zero\n    f1 = np.nan_to_num(f1)\n\n    # compute micro-average f1 score\n    micro_f1 = np.mean(f1)\n    # Return mean F1 score across all samples\n    return micro_f1\n\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T10:20:55.489991Z","iopub.execute_input":"2024-07-09T10:20:55.490382Z","iopub.status.idle":"2024-07-09T10:20:55.498012Z","shell.execute_reply.started":"2024-07-09T10:20:55.490350Z","shell.execute_reply":"2024-07-09T10:20:55.497110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if cuda is available\ndevice = torch.device(\"cpu\")\n\nif torch.cuda.is_available():\n    device = torch.device(\"cuda\")\n    \n\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T10:20:55.888615Z","iopub.execute_input":"2024-07-09T10:20:55.888957Z","iopub.status.idle":"2024-07-09T10:20:55.894078Z","shell.execute_reply.started":"2024-07-09T10:20:55.888931Z","shell.execute_reply":"2024-07-09T10:20:55.893235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define model\n\nimport torch.nn.functional as F\n\nclass MultimodalEnsemble(nn.Module):\n    def __init__(self):\n        super(MultimodalEnsemble, self).__init__()\n        \n        self.landsat_norm = nn.LayerNorm([6,4,21])\n        self.landsat_model = models.resnet18(weights=None)\n        # Modify the first convolutional layer to accept 6 channels instead of 3\n        self.landsat_model.conv1 = nn.Conv2d(6, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.landsat_model.maxpool = nn.Identity()\n        \n        self.bioclim_norm = nn.LayerNorm([4,19,12])\n        self.bioclim_model = models.resnet18(weights=None)  \n        # Modify the first convolutional layer to accept 4 channels instead of 3\n        self.bioclim_model.conv1 = nn.Conv2d(4, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bioclim_model.maxpool = nn.Identity()\n        \n        self.sentinel_model = models.swin_t(weights=\"IMAGENET1K_V1\")\n        # Modify the first layer to accept 4 channels instead of 3\n        self.sentinel_model.features[0][0] = nn.Conv2d(4, 96, kernel_size=(4, 4), stride=(4, 4))\n        self.sentinel_model.head = nn.Identity()\n        \n        self.ln1 = nn.LayerNorm(1000)\n        self.ln2 = nn.LayerNorm(1000)\n        \n        \n    def forward(self, x, y, z):\n        \n        x = self.landsat_norm(x)\n        x = self.landsat_model(x)\n        x = self.ln1(x)\n        \n        y = self.bioclim_norm(y)\n        y = self.bioclim_model(y)\n        y = self.ln2(y)\n        \n        z = self.sentinel_model(z)\n        \n        out = torch.cat((x, y, z), dim=1)\n        \n        return out\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T10:20:56.127284Z","iopub.execute_input":"2024-07-09T10:20:56.127832Z","iopub.status.idle":"2024-07-09T10:20:56.138754Z","shell.execute_reply.started":"2024-07-09T10:20:56.127806Z","shell.execute_reply":"2024-07-09T10:20:56.137866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop with custom F1 score\nf1s_knn = []\n\nfor i in range(n):\n\n    print(f\"Iteration: {i+1}\")\n\n    # get train and val survey ids\n    train_survey_ids, val_survey_ids = get_train_val_survey_ids(survey_ids, train_ratio)\n\n    train_loader, val_loader = create_loaders(train_survey_ids, val_survey_ids)\n    \n    feature_extractor = MultimodalEnsemble().to(device)\n\n    # Training\n    print('Extracting features...')\n\n    feature_extractor.train()\n    \n    # Feature Extraction\n    X = []\n    Y = []\n    for data1, data2, data3, target , _ in tqdm(train_loader):\n        data1 = data1.to(device)\n        data2 = data2.to(device)\n        data3 = data3.to(device)\n        feature_vector = feature_extractor(data1, data2, data3).cpu().detach().numpy()\n\n        X.append(feature_vector)\n        Y.append(target)\n        \n    print('X:', len(X), len(X[0]), len(X[0][0]))\n    print('Y:',len(X), len(X[0]), len(X[0][0]))\n        \n    X = np.concatenate(X, axis=0)\n    Y = np.concatenate(Y, axis=0)\n    \n    print('X:',X.shape)\n    print('X:',Y.shape)\n\n    print('Shape of feature dataset:', )\n    # instantiate a NearestNeighbors object\n    nn_model = NearestNeighbors(n_neighbors=5, metric='euclidean')\n\n    # fit model on dataset\n    print('Fitting kNN...')\n    nn_model.fit(X)\n\n    # Predict with trained models on validation set\n    feature_extractor.eval()\n    Y_preds = []\n    Y_trues = []\n    print('Validating model...')\n    for data1, data2, data3, targets, _ in tqdm(val_loader):\n        data1 = data1.to(device)\n        data2 = data2.to(device)\n        data3 = data3.to(device)\n        outputs = feature_extractor(data1, data2, data3).cpu().detach().numpy()\n\n        for output in outputs:\n            distances, indices = nn_model.kneighbors(output.reshape(1, -1))\n            y_neighbors = Y[indices[0], :]\n            avg = y_neighbors.mean(axis=0).round()\n            Y_preds.append(avg)\n        Y_trues.append(targets.cpu().detach().numpy())\n\n    Y_preds = np.vstack(Y_preds).astype(int)\n    Y_trues = np.vstack(Y_trues).astype(int)\n    print(Y_preds.shape, Y_trues.shape)\n    \n\n\n    # Calculate custom F1 score\n    f1 = calculate_f1_score_from_tensors(Y_trues, Y_preds, threshold=0.5)\n        \n    print(f\"Iteration {i+1} - Mean F1 Score: {f1}\")\n\n    # Append mean F1 score to list\n    f1s_knn.append(f1)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T11:03:22.304280Z","iopub.execute_input":"2024-07-09T11:03:22.305475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\n\nnow = datetime.datetime.now()\ntimestamp = now.strftime('%Y-%m-%d_%H-%M-%S')\n\nresults = pd.DataFrame(f1s_knn, columns=['knn_f1'])\nresults.to_csv(f'{timestamp}_knn_benchmark_results_{n}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T10:26:50.110056Z","iopub.status.idle":"2024-07-09T10:26:50.110409Z","shell.execute_reply.started":"2024-07-09T10:26:50.110239Z","shell.execute_reply":"2024-07-09T10:26:50.110257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(Y_preds.shape, Y_trues.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T10:29:11.907796Z","iopub.execute_input":"2024-07-09T10:29:11.908409Z","iopub.status.idle":"2024-07-09T10:29:11.913573Z","shell.execute_reply.started":"2024-07-09T10:29:11.908376Z","shell.execute_reply":"2024-07-09T10:29:11.912673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}