{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":64733,"databundleVersionId":8171035,"sourceType":"competition"},{"sourceId":43782,"sourceType":"modelInstanceVersion","modelInstanceId":36765}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is an INFERENCE using the model combine the method of MLP, CNN and Transformer\n\nThanks PICEKL for his baseline, which help me a lot.\n\nThe main idea of this method is to obtain multiple features from multiple dimensions. We simply divide it into four types: **image information** with spatial characteristics, **meta information** that is independent of each other, **climate information** and **satellite information** with time characteristics.","metadata":{}},{"cell_type":"code","source":"! pip install einops","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:32:55.283754Z","iopub.execute_input":"2024-05-05T13:32:55.284144Z","iopub.status.idle":"2024-05-05T13:33:11.641042Z","shell.execute_reply.started":"2024-05-05T13:32:55.284112Z","shell.execute_reply":"2024-05-05T13:33:11.639524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport tqdm\nimport numpy as np\nimport pandas as pd\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom sklearn.metrics import precision_recall_fscore_support\nfrom PIL import Image\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom einops.layers.torch import Rearrange\nfrom einops import rearrange, repeat","metadata":{"ExecuteTime":{"end_time":"2024-04-30T21:25:07.29831Z","start_time":"2024-04-30T21:25:05.354584Z"},"tags":[],"execution":{"iopub.status.busy":"2024-05-05T13:33:11.643548Z","iopub.execute_input":"2024-05-05T13:33:11.643932Z","iopub.status.idle":"2024-05-05T13:33:18.649546Z","shell.execute_reply.started":"2024-05-05T13:33:11.643899Z","shell.execute_reply":"2024-05-05T13:33:18.64851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 11255","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:33:18.650749Z","iopub.execute_input":"2024-05-05T13:33:18.651398Z","iopub.status.idle":"2024-05-05T13:33:18.655878Z","shell.execute_reply.started":"2024-05-05T13:33:18.651368Z","shell.execute_reply":"2024-05-05T13:33:18.654845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"When reading data, different **fusion** and **normalization** will be performed for different types of data.","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, metadata, subset, transform=None):\n        self.subset = subset\n        self.transform = transform\n        self.metadata = metadata\n        \n        self.metadata = self.metadata.drop_duplicates(subset=\"surveyId\").reset_index(drop=True)\n        self.metadata.fillna(0,inplace=True)\n        self.metadata.replace({float('-inf'): 0}, inplace=True)\n        self.metadata_data = self.Norm(self.metadata.iloc[:,:5])\n        \n        self.merge_key = 'surveyId'\n        self.climate_average = pd.read_csv(\"/kaggle/input/geolifeclef-2024/EnvironmentalRasters/EnvironmentalRasters/Climate/Average 1981-2010/GLC24-PA-test-bioclimatic.csv\")\n        self.climate_monthly = pd.read_csv(\"/kaggle/input/geolifeclef-2024/EnvironmentalRasters/EnvironmentalRasters/Climate/Monthly/GLC24-PA-test-bioclimatic_monthly.csv\")\n        self.climate = pd.merge(self.climate_average, self.climate_monthly, on=self.merge_key)\n        self.climate.fillna(self.climate.mean(),inplace=True)\n        self.climate_data = self.Norm_all(self.climate)\n        \n        self.landsat_b = pd.read_csv(\"/kaggle/input/geolifeclef-2024/PA-test-landsat_time_series/GLC24-PA-test-landsat_time_series-blue.csv\")\n        self.landsat_b.fillna(self.landsat_b.mean(),inplace=True)\n        self.landsat_g = pd.read_csv(\"/kaggle/input/geolifeclef-2024/PA-test-landsat_time_series/GLC24-PA-test-landsat_time_series-green.csv\")\n        self.landsat_g.fillna(self.landsat_g.mean(),inplace=True)\n        self.landsat_r = pd.read_csv(\"/kaggle/input/geolifeclef-2024/PA-test-landsat_time_series/GLC24-PA-test-landsat_time_series-red.csv\")\n        self.landsat_r.fillna(self.landsat_r.mean(),inplace=True)\n        self.landsat_n = pd.read_csv(\"/kaggle/input/geolifeclef-2024/PA-test-landsat_time_series/GLC24-PA-test-landsat_time_series-nir.csv\")\n        self.landsat_n.fillna(self.landsat_n.mean(),inplace=True)\n        self.landsat_s1 = pd.read_csv(\"/kaggle/input/geolifeclef-2024/PA-test-landsat_time_series/GLC24-PA-test-landsat_time_series-swir1.csv\")\n        self.landsat_s1.fillna(self.landsat_s1.mean(),inplace=True)\n        self.landsat_s2 = pd.read_csv(\"/kaggle/input/geolifeclef-2024/PA-test-landsat_time_series/GLC24-PA-test-landsat_time_series-swir2.csv\")\n        self.landsat_s2.fillna(self.landsat_s2.mean(),inplace=True)\n        self.landsat_data = torch.cat([self.Norm_all(self.landsat_b),self.Norm_all(self.landsat_g),self.Norm_all(self.landsat_r),self.Norm_all(self.landsat_n),self.Norm_all(self.landsat_s1),self.Norm_all(self.landsat_s2)],axis=1)\n         \n        self.elevation = pd.read_csv(\"/kaggle/input/geolifeclef-2024/EnvironmentalRasters/EnvironmentalRasters/Elevation/GLC24-PA-test-elevation.csv\")\n        self.elevation[self.elevation<0]=0\n        self.elevation.fillna(self.elevation.mean(),inplace=True)\n        self.elevation_data = self.Norm(self.elevation)\n        \n        self.human_footprint = pd.read_csv(\"/kaggle/input/geolifeclef-2024/EnvironmentalRasters/EnvironmentalRasters/Human Footprint/GLC24-PA-test-human_footprint.csv\")\n        self.human_footprint[self.human_footprint<0]=0\n        self.human_footprint.fillna(self.human_footprint.mean(),inplace=True)  \n        self.human_footprint_data = self.Norm(self.human_footprint)\n        \n        self.landcover = pd.read_csv(\"/kaggle/input/geolifeclef-2024/EnvironmentalRasters/EnvironmentalRasters/LandCover/GLC24-PA-test-landcover.csv\")\n        self.landcover[self.landcover<0]=0\n        self.landcover.fillna(self.landcover.mean(),inplace=True)    \n        self.landcover_data = self.Norm(self.landcover)\n        \n        self.soilgrids = pd.read_csv(\"/kaggle/input/geolifeclef-2024/EnvironmentalRasters/EnvironmentalRasters/SoilGrids/GLC24-PA-test-soilgrids.csv\")\n        self.soilgrids[self.soilgrids<0]=0\n        self.soilgrids.fillna(self.soilgrids.mean(),inplace=True)\n        self.soilgrids_data = self.Norm(self.soilgrids)\n        \n        self.metadata_data = torch.cat((self.metadata_data, self.elevation_data, self.human_footprint_data, self.landcover_data, self.soilgrids_data), dim=1)\n        \n    def Norm(self,df):\n        output=torch.from_numpy(df.iloc[:,1:].values).float()\n        return (output-output.mean(dim=0))/output.std(dim=0)\n    \n    def Norm_all(self,df):\n        output=torch.from_numpy(df.iloc[:,1:].values).float()\n        return (output-output.mean())/output.std()\n    \n    def patch_rgb_path(self,survey_id):\n        path = \"/kaggle/input/geolifeclef-2024/PA_Test_SatellitePatches_RGB/pa_test_patches_rgb\"\n        for d in (str(survey_id)[-2:], str(survey_id)[-4:-2]):\n            path = os.path.join(path, d)\n        path = os.path.join(path, f\"{survey_id}.jpeg\")\n        return path\n    \n    def patch_nir_path(self,survey_id):\n        path = \"/kaggle/input/geolifeclef-2024/PA_Test_SatellitePatches_NIR/pa_test_patches_nir\"\n        for d in (str(survey_id)[-2:], str(survey_id)[-4:-2]):\n            path = os.path.join(path, d)\n        path = os.path.join(path, f\"{survey_id}.jpeg\")\n        return path\n\n    def __len__(self):\n        return len(self.metadata)\n\n    def __getitem__(self, idx):\n        \n        survey_id = self.metadata.surveyId[idx]\n        \n        image_path = self.patch_rgb_path(survey_id)\n        image = Image.open(image_path).convert(\"RGB\")\n        image = self.transform(image)\n        image = image.unsqueeze(0)\n        image_nir_path = self.patch_nir_path(survey_id)\n        nir_image = Image.open(image_nir_path).convert(\"L\")\n        nir_image = self.transform(nir_image)\n        nir_image = nir_image.unsqueeze(0)\n        image_data = torch.cat([image,nir_image],dim=1)\n        image_data = torch.squeeze(image_data)\n        sample=[self.metadata_data[idx,:],image_data,self.landsat_data[idx,:],self.climate_data[idx,:]]\n        return sample, survey_id","metadata":{"ExecuteTime":{"end_time":"2024-04-30T21:25:32.627928Z","start_time":"2024-04-30T21:25:32.612131Z"},"collapsed":false,"jupyter":{"outputs_hidden":false},"tags":[],"execution":{"iopub.status.busy":"2024-05-05T13:33:18.659314Z","iopub.execute_input":"2024-05-05T13:33:18.659742Z","iopub.status.idle":"2024-05-05T13:33:18.695531Z","shell.execute_reply.started":"2024-05-05T13:33:18.659704Z","shell.execute_reply":"2024-05-05T13:33:18.69374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset and DataLoader\nbatch_size = 1\ntransform = transforms.Compose([\n    transforms.Resize((128,128)),\n    transforms.ToTensor()\n])\n\n# Load Training metadata\ntest_metadata_path = \"/kaggle/input/geolifeclef-2024/GLC24_PA_metadata_test.csv\"\ntest_metadata = pd.read_csv(test_metadata_path)\ntest_dataset = TestDataset(test_metadata, subset=\"test\", transform=transform)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=1)","metadata":{"ExecuteTime":{"end_time":"2024-04-30T21:25:34.532017Z","start_time":"2024-04-30T21:25:32.615562Z"},"collapsed":false,"jupyter":{"outputs_hidden":false},"tags":[],"execution":{"iopub.status.busy":"2024-05-05T13:35:39.876819Z","iopub.execute_input":"2024-05-05T13:35:39.877256Z","iopub.status.idle":"2024-05-05T13:35:41.609946Z","shell.execute_reply.started":"2024-05-05T13:35:39.877218Z","shell.execute_reply":"2024-05-05T13:35:41.608959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is an MLP used to extract features from generally independent information.","metadata":{}},{"cell_type":"code","source":"class Embedding(nn.Module):\n    def __init__(self, dim, out_dim):\n        super().__init__()\n        self.fc1 = nn.Linear(dim, out_dim*5)\n        self.fc2 = nn.Linear(out_dim*5, out_dim)\n        self.norm = nn.LayerNorm(out_dim*5)\n\n    def forward(self, x):\n        x = F.tanh(self.fc1(x))\n        x = self.norm(x)\n        x = self.fc2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:33:21.01634Z","iopub.execute_input":"2024-05-05T13:33:21.016671Z","iopub.status.idle":"2024-05-05T13:33:21.025439Z","shell.execute_reply.started":"2024-05-05T13:33:21.016629Z","shell.execute_reply":"2024-05-05T13:33:21.022979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following is the part of ViT, which can also be considered as the Encoder part of **Transformer**.","metadata":{}},{"cell_type":"code","source":"class Multihead_self_attention(nn.Module):\n    def __init__(self, heads, head_dim, dim):\n        super().__init__()\n        self.head_dim = head_dim\n        self.heads = heads \n        self.inner_dim = self.heads*self.head_dim  \n        self.scale = self.head_dim**-0.5   \n        self.to_qkv = nn.Linear(dim, self.inner_dim*3) \n        self.to_output = nn.Linear(self.inner_dim, dim)\n        self.norm = nn.LayerNorm(dim)\n        self.softmax = nn.Softmax(dim=-1)\n\n    def forward(self, x):\n        x = self.norm(x)    \n        qkv = self.to_qkv(x).chunk(3, dim=-1) \n        Q, K, V = map(lambda t: rearrange(t, 'b l (h dim) -> b h l dim', dim=self.head_dim), qkv)\n        K_T = K.transpose(-1, -2)\n        att_score = Q@K_T*self.scale\n        att = self.softmax(att_score)\n        out = att@V   # (B,H,L,dim)\n        out = rearrange(out, 'b h l dim -> b l (h dim)')  \n        output = self.to_output(out)\n        return output","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:33:21.026968Z","iopub.execute_input":"2024-05-05T13:33:21.027395Z","iopub.status.idle":"2024-05-05T13:33:21.03911Z","shell.execute_reply.started":"2024-05-05T13:33:21.027364Z","shell.execute_reply":"2024-05-05T13:33:21.037883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedForward(nn.Module):\n    def __init__(self, dim, mlp_dim):\n        super().__init__()\n        self.fc1 = nn.Linear(dim, mlp_dim)\n        self.fc2 = nn.Linear(mlp_dim, dim)\n        self.norm = nn.LayerNorm(dim)\n\n    def forward(self, x):\n        x = self.norm(x)\n        x = F.gelu(self.fc1(x))\n        x = self.fc2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:33:21.040684Z","iopub.execute_input":"2024-05-05T13:33:21.041013Z","iopub.status.idle":"2024-05-05T13:33:21.058188Z","shell.execute_reply.started":"2024-05-05T13:33:21.040986Z","shell.execute_reply":"2024-05-05T13:33:21.056884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Transformer_block(nn.Module):\n    def __init__(self, dim, heads, head_dim, mlp_dim):\n        super().__init__()\n        self.MHA = Multihead_self_attention(heads=heads, head_dim=head_dim, dim=dim)\n        self.FeedForward = FeedForward(dim=dim, mlp_dim=mlp_dim)\n\n    def forward(self, x):\n        x = self.MHA(x)+x\n        x = self.FeedForward(x)+x\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:33:21.059838Z","iopub.execute_input":"2024-05-05T13:33:21.060277Z","iopub.status.idle":"2024-05-05T13:33:21.070067Z","shell.execute_reply.started":"2024-05-05T13:33:21.060243Z","shell.execute_reply":"2024-05-05T13:33:21.068822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ViT(nn.Module):\n    def __init__(self, dim, heads, head_dim, mlp_dim, num_class):\n        super().__init__()\n        self.transformer = Transformer_block(dim=dim, heads=heads, head_dim=head_dim, mlp_dim=mlp_dim)\n\n        self.MLP_head = nn.Sequential(\n            nn.LayerNorm(dim),\n            nn.Linear(dim, num_class)\n        )\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        x = self.transformer(x)\n        CLS_token = x[:, 0, :] \n        out = self.MLP_head(CLS_token)\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:33:21.07438Z","iopub.execute_input":"2024-05-05T13:33:21.074785Z","iopub.status.idle":"2024-05-05T13:33:21.084142Z","shell.execute_reply.started":"2024-05-05T13:33:21.074752Z","shell.execute_reply":"2024-05-05T13:33:21.082968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following is **CNN**, used to extract feature information from images.","metadata":{}},{"cell_type":"code","source":"class ResNet18(nn.Module):\n    def __init__(self, num_classes):\n        super(ResNet18, self).__init__()\n\n        self.resnet18 = models.resnet18(weights=None)\n        self.resnet18.conv1 = nn.Conv2d(4, 64, kernel_size=3, stride=1, padding=1, bias=False)\n        self.resnet18.maxpool = nn.Identity()\n        self.ln = nn.LayerNorm(1000)\n        self.fc1 = nn.Linear(1000, num_classes)\n\n    def forward(self, x):\n        x = self.resnet18(x)\n        x = self.ln(x)\n        x = self.fc1(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:33:21.085836Z","iopub.execute_input":"2024-05-05T13:33:21.086503Z","iopub.status.idle":"2024-05-05T13:33:21.098855Z","shell.execute_reply.started":"2024-05-05T13:33:21.086461Z","shell.execute_reply":"2024-05-05T13:33:21.097821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is the **final multi-modal model**. Unlike ViT, its input is features extracted from each dimension. \n\nFor features with time information, I added additional position information. \n\nAt the same time, I also added a feature that is a fusion of the first four features.","metadata":{}},{"cell_type":"code","source":"class MutiModal(nn.Module):\n    def __init__(self, num_classes):\n        super(MutiModal, self).__init__()\n        self.cls = nn.Parameter(torch.randn(1, 1, 200))\n        self.meta = Embedding(31,200)\n        self.resnet18 = ResNet18(200)\n        self.landsat = Embedding(504,200)\n        self.position_landsat = nn.Parameter(torch.randn(1, 504))\n        self.climate = Embedding(931,200)\n        self.position_climate = nn.Parameter(torch.randn(1, 931))\n        self.emb = Embedding(800,200)\n        self.position_combine = nn.Parameter(torch.randn(1, 800))\n        self.vit = ViT(200, 2, 200, 400, num_classes)\n        self.position = nn.Parameter(torch.randn(1, 6, 200))\n\n    def forward(self, x):\n        batch = x[0].size(0)\n        CLS = repeat(self.cls, '1 1 d -> b 1 d', b=batch).to(device)\n        META = self.meta(x[0])\n        IMG = self.resnet18(x[1])\n        LANDSAT = self.landsat(x[2]+self.position_landsat)\n        CLIMATE = self.climate(x[3]+self.position_climate)\n        combine = torch.cat((META, IMG, LANDSAT, CLIMATE), dim=1)\n        COMBINE = self.emb(combine+self.position_combine)\n        token = torch.concat((CLS, META.unsqueeze(1)), dim=1)\n        token = torch.concat((token, IMG.unsqueeze(1)), dim=1)\n        token = torch.concat((token, LANDSAT.unsqueeze(1)), dim=1)\n        token = torch.concat((token, CLIMATE.unsqueeze(1)), dim=1)\n        token = torch.concat((token, COMBINE.unsqueeze(1)), dim=1)\n        out = self.vit(token+self.position)\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:33:21.10075Z","iopub.execute_input":"2024-05-05T13:33:21.101145Z","iopub.status.idle":"2024-05-05T13:33:21.117011Z","shell.execute_reply.started":"2024-05-05T13:33:21.101114Z","shell.execute_reply":"2024-05-05T13:33:21.115715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if cuda is available\ndevice = torch.device(\"cpu\")\n\nif torch.cuda.is_available():\n    device = torch.device(\"cuda\")\n    print(\"DEVICE = CUDA\")\n\nmodel = MutiModal(num_classes).to(device)\nmodel.load_state_dict(torch.load(\"/kaggle/input/mutimodal-vit/pytorch/mvit/1/Model.pth\", map_location=device))","metadata":{"ExecuteTime":{"end_time":"2024-04-30T21:25:31.611823Z","start_time":"2024-04-30T21:25:31.607373Z"},"collapsed":false,"jupyter":{"outputs_hidden":false},"tags":[],"execution":{"iopub.status.busy":"2024-05-05T13:33:21.118365Z","iopub.execute_input":"2024-05-05T13:33:21.118714Z","iopub.status.idle":"2024-05-05T13:33:22.163616Z","shell.execute_reply.started":"2024-05-05T13:33:21.118665Z","shell.execute_reply":"2024-05-05T13:33:22.16128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    surveys = []\n    top_indices = []\n    for data, surveyID in tqdm.tqdm(test_loader, total=len(test_loader)):\n\n        data = [tensor.to(device) for tensor in data]\n        \n        outputs = model(data)\n        predictions = torch.sigmoid(outputs).cpu().numpy()\n        predictions = np.squeeze(predictions)\n        prediction = np.argwhere(predictions>=0.95).flatten()\n        top_indices.append(prediction)\n        surveys.extend(surveyID.cpu().numpy())","metadata":{"ExecuteTime":{"start_time":"2024-04-30T21:25:34.536634Z"},"collapsed":false,"is_executing":true,"jupyter":{"outputs_hidden":false},"tags":[],"execution":{"iopub.status.busy":"2024-05-05T13:51:04.312257Z","iopub.execute_input":"2024-05-05T13:51:04.312679Z","iopub.status.idle":"2024-05-05T13:51:08.691165Z","shell.execute_reply.started":"2024-05-05T13:51:04.312632Z","shell.execute_reply":"2024-05-05T13:51:08.689384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_concatenated = [' '.join(map(str, row)) for row in top_indices]\n\npd.DataFrame(\n    {'surveyId': surveys,\n     'predictions': data_concatenated,\n    }).to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2024-05-05T13:51:27.185326Z","iopub.execute_input":"2024-05-05T13:51:27.186452Z","iopub.status.idle":"2024-05-05T13:51:27.196884Z","shell.execute_reply.started":"2024-05-05T13:51:27.186405Z","shell.execute_reply":"2024-05-05T13:51:27.19557Z"},"trusted":true},"execution_count":null,"outputs":[]}]}