{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":64733,"databundleVersionId":8171035,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\ntry:\n    from torch_geometric.data import Data\nexcept:\n    !pip install torch-geometric\n    \n    \nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport torch\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch_geometric.data import Data\nfrom PIL import Image\nfrom tqdm import tqdm\nimport torch_cluster\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-05T06:48:41.493726Z","iopub.execute_input":"2024-06-05T06:48:41.494193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def construct_patch_path(output_path, survey_id):\n    \"\"\"Construct the patch file path based on survey_id as './CD/AB/XXXXABCD.jpeg'\"\"\"\n    path = output_path\n    for d in (str(survey_id)[-2:], str(survey_id)[-4:-2]):\n        path = os.path.join(path, d)\n\n    path = os.path.join(path, f\"{survey_id}.jpeg\")\n\n    return path\n\n\nclass TrainDataset(Dataset):\n    def __init__(self, bioclim_data_dir, landsat_data_dir, sentinel_data_dir, metadata, transform=None):\n        self.transform = transform\n        self.sentinel_transform = transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize(mean=(0.5, 0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5, 0.5)),\n        ])\n      \n        self.bioclim_data_dir = bioclim_data_dir\n        self.landsat_data_dir = landsat_data_dir\n        self.sentinel_data_dir = sentinel_data_dir\n        self.metadata = metadata\n        self.metadata = self.metadata.dropna(subset=\"speciesId\").reset_index(drop=True)\n        self.metadata['speciesId'] = self.metadata['speciesId'].astype(int)\n        self.label_dict = self.metadata.groupby('surveyId')['speciesId'].apply(list).to_dict()\n        \n        self.metadata = self.metadata.drop_duplicates(subset=\"surveyId\").reset_index(drop=True)\n\n    def __len__(self):\n        return len(self.metadata)\n\n    def __getitem__(self, idx):\n        \n        survey_id = self.metadata.surveyId[idx]\n        \n        landsat_sample = torch.nan_to_num(torch.load(os.path.join(self.landsat_data_dir, f\"GLC24-PA-train-landsat-time-series_{survey_id}_cube.pt\")))\n        bioclim_sample = torch.nan_to_num(torch.load(os.path.join(self.bioclim_data_dir, f\"GLC24-PA-train-bioclimatic_monthly_{survey_id}_cube.pt\")))\n\n        rgb_sample = np.array(Image.open(construct_patch_path(self.sentinel_data_dir, survey_id)))\n        nir_sample = np.array(Image.open(construct_patch_path(self.sentinel_data_dir.replace(\"rgb\", \"nir\").replace(\"RGB\", \"NIR\"), survey_id)))\n        sentinel_sample = np.concatenate((rgb_sample, nir_sample[...,None]), axis=2)\n\n        \n        if isinstance(landsat_sample, torch.Tensor):\n            landsat_sample = landsat_sample.permute(1, 2, 0)  # Change tensor shape from (C, H, W) to (H, W, C)\n            landsat_sample = landsat_sample.numpy()  # Convert tensor to numpy array\n            \n        if isinstance(bioclim_sample, torch.Tensor):\n            bioclim_sample = bioclim_sample.permute(1, 2, 0)  # Change tensor shape from (C, H, W) to (H, W, C)\n            bioclim_sample = bioclim_sample.numpy()  # Convert tensor to numpy array   \n        \n        if self.transform:\n            landsat_sample = self.transform(landsat_sample)\n            bioclim_sample = self.transform(bioclim_sample)\n            sentinel_sample = self.sentinel_transform(sentinel_sample)\n\n        return landsat_sample, bioclim_sample, sentinel_sample","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset and DataLoader\nbatch_size = 128\ntransform = transforms.Compose([\n    transforms.ToTensor()\n])\n\n\n# Load Training metadata\ntrain_landsat_data_path = \"/kaggle/input/geolifeclef-2024/TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-landsat_time_series\"\ntrain_bioclim_data_path = \"/kaggle/input/geolifeclef-2024/TimeSeries-Cubes/TimeSeries-Cubes/GLC24-PA-train-bioclimatic_monthly\"\ntrain_sentinel_data_path = \"/kaggle/input/geolifeclef-2024/PA_Train_SatellitePatches_RGB/pa_train_patches_rgb\"\ntrain_metadata_path = \"/kaggle/input/geolifeclef-2024/GLC24_PA_metadata_train.csv\"\n\ntrain_metadata = pd.read_csv(train_metadata_path)\n\ntrain_dataset = TrainDataset(train_bioclim_data_path, train_landsat_data_path, train_sentinel_data_path, train_metadata, transform=transform)\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F\n\nclass MultimodalEmbedder(nn.Module):\n    def __init__(self):\n        super(MultimodalEmbedder, self).__init__()\n        \n        self.landsat_norm = nn.LayerNorm([6,4,21])\n        self.landsat_model = models.resnet18(weights=None)\n        # Modify the first convolutional layer to accept 6 channels instead of 3\n        self.landsat_model.conv1 = nn.Conv2d(6, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.landsat_model.maxpool = nn.Identity()\n        \n        self.bioclim_norm = nn.LayerNorm([4,19,12])\n        self.bioclim_model = models.resnet18(weights=None)  \n        # Modify the first convolutional layer to accept 4 channels instead of 3\n        self.bioclim_model.conv1 = nn.Conv2d(4, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bioclim_model.maxpool = nn.Identity()\n        \n        self.sentinel_model = models.swin_t(weights=\"IMAGENET1K_V1\")\n        # Modify the first layer to accept 4 channels instead of 3\n        self.sentinel_model.features[0][0] = nn.Conv2d(4, 96, kernel_size=(4, 4), stride=(4, 4))\n        self.sentinel_model.head = nn.Identity()\n\n        self.ln1 = nn.LayerNorm(1000)\n        self.ln2 = nn.LayerNorm(1000)\n        \n        \n    def forward(self, x, y, z):\n        \n        x = self.landsat_norm(x)\n        x = self.landsat_model(x)\n        x = self.ln1(x)\n        \n        y = self.bioclim_norm(y)\n        y = self.bioclim_model(y)\n        y = self.ln2(y)\n        \n        z = self.sentinel_model(z)\n        \n        xyz = torch.cat((x, y, z), dim=1)\n        return xyz","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    # Set seed for Python's built-in random number generator\n    torch.manual_seed(seed)\n    # Set seed for numpy\n    np.random.seed(seed)\n    # Set seed for CUDA if available\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        # Set cuDNN's random number generator seed for deterministic behavior\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\nset_seed(69)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if cuda is available\ndevice = torch.device(\"cpu\")\n\nif torch.cuda.is_available():\n    device = torch.device(\"cuda\")\n    print(\"DEVICE = CUDA\")\n\nmodel = MultimodalEmbedder().to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Embedding process started.\")\n\n\nembeddings = []\n\nmodel.train()\n\nfor batch_idx, (landsat, bioclim, sentinel) in tqdm(enumerate(train_loader)):\n\n    landsat = landsat.to(device)\n    bioclim = bioclim.to(device)\n    sentinel = sentinel.to(device)\n\n    #optimizer.zero_grad()\n    outputs = model(landsat, bioclim, sentinel)\n     # Move outputs to CPU and detach from computational graph\n    outputs_cpu = outputs.detach().cpu()\n    \n    # Append outputs to embeddings list\n    embeddings.append(outputs_cpu)\n    \n    # Clear GPU memory\n    del outputs\n    torch.cuda.empty_cache()\n    \nprint(f\"Embedding process completed.\")\nprint(len(embeddings), len(embeddings[0]), len(embeddings[0][0]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Convert list of numpy arrays to a single numpy array\nembeddings_array = np.concatenate(embeddings, axis=0)  # Concatenate along the first dimension\n\n# Check the shape of the final array\nprint(f\"Final embeddings array shape: {embeddings_array.shape}\")\n\n# Convert back to a torch tensor\nembeddings_tensor = torch.tensor(embeddings_array)\n\n# Save the tensor to a .pt file\ntorch.save(embeddings_tensor, 'embeddings.pt')\n\nprint(f\"Saved tensor with shape {embeddings_tensor.shape} to embeddings.pt\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}