{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "36589df7", "metadata": { "execution": { "iopub.execute_input": "2023-12-13T04:02:54.310434Z", "iopub.status.busy": "2023-12-13T04:02:54.310096Z", "iopub.status.idle": "2023-12-13T04:03:00.188617Z", "shell.execute_reply": "2023-12-13T04:03:00.187689Z" }, "papermill": { "duration": 5.886002, "end_time": "2023-12-13T04:03:00.191067", "exception": false, "start_time": "2023-12-13T04:02:54.305065", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "import warnings\n", "from typing import List, Optional, Tuple, Union\n", "\n", "import torch\n", "from torch import nn\n", "\n", "\n", "class MLP(torch.nn.Sequential):\n", " \"\"\"MLP Module.\n", "\n", " Parameters\n", " ----------\n", " in_features: int\n", " Features (model input) dimension.\n", " out_features: int = 1\n", " Prediction (model output) dimension.\n", " hidden: Optional[List[int]] = None\n", " Dimension of hidden layer(s).\n", " dropout: Optional[List[float]] = None\n", " Dropout rate(s).\n", " activation: Optional[torch.nn.Module] = torch.nn.Sigmoid\n", " MLP activation.\n", " bias: bool = True\n", " Add bias to MLP hidden layers.\n", "\n", " Raises\n", " ------\n", " ValueError\n", " If ``hidden`` and ``dropout`` do not share the same length.\n", " \"\"\"\n", "\n", " def __init__(\n", " self,\n", " in_features: int,\n", " out_features: int,\n", " hidden: Optional[List[int]] = None,\n", " dropout: Optional[List[float]] = None,\n", " activation: Optional[torch.nn.Module] = torch.nn.Sigmoid(),\n", " bias: bool = True,\n", " ):\n", " if dropout is not None:\n", " if hidden is not None:\n", " assert len(hidden) == len(\n", " dropout\n", " ), \"hidden and dropout must have the same length\"\n", " else:\n", " raise ValueError(\n", " \"hidden must have a value and have the same length as dropout if dropout is given.\"\n", " )\n", "\n", " d_model = in_features\n", " layers = []\n", "\n", " if hidden is not None:\n", " for i, h in enumerate(hidden):\n", " seq = [torch.nn.Linear(d_model, h, bias=bias)]\n", " d_model = h\n", "\n", " if activation is not None:\n", " seq.append(activation)\n", "\n", " if dropout is not None:\n", " seq.append(torch.nn.Dropout(dropout[i]))\n", "\n", " layers.append(torch.nn.Sequential(*seq))\n", "\n", " layers.append(torch.nn.Linear(d_model, out_features))\n", "\n", " super(MLP, self).__init__(*layers)\n", "\n", "\n", "class MaskedLinear(torch.nn.Linear):\n", " \"\"\"\n", " Linear layer to be applied tile wise.\n", " This layer can be used in combination with a mask\n", " to prevent padding tiles from influencing the values of a subsequent\n", " activation.\n", " Example:\n", " >>> module = Linear(in_features=128, out_features=1) # With Linear\n", " >>> out = module(slide)\n", " >>> wrong_value = torch.sigmoid(out) # Value is influenced by padding\n", " >>> module = MaskedLinear(in_features=128, out_features=1, mask_value='-inf') # With MaskedLinear\n", " >>> out = module(slide, mask) # Padding now has the '-inf' value\n", " >>> correct_value = torch.sigmoid(out) # Value is not influenced by padding as sigmoid('-inf') = 0\n", " Parameters\n", " ----------\n", " in_features: int\n", " size of each input sample\n", " out_features: int\n", " size of each output sample\n", " mask_value: Union[str, int]\n", " value to give to the mask\n", " bias: bool = True\n", " If set to ``False``, the layer will not learn an additive bias.\n", " \"\"\"\n", "\n", " def __init__(\n", " self,\n", " in_features: int,\n", " out_features: int,\n", " mask_value: Union[str, float],\n", " bias: bool = True,\n", " ):\n", " super(MaskedLinear, self).__init__(\n", " in_features=in_features, out_features=out_features, bias=bias\n", " )\n", " self.mask_value = mask_value\n", "\n", " def forward(\n", " self, x: torch.Tensor, mask: Optional[torch.BoolTensor] = None\n", " ): # pylint: disable=arguments-renamed\n", " \"\"\"Forward pass.\n", "\n", " Parameters\n", " ----------\n", " x: torch.Tensor\n", " Input tensor, shape (B, SEQ_LEN, IN_FEATURES).\n", " mask: Optional[torch.BoolTensor] = None\n", " True for values that were padded, shape (B, SEQ_LEN, 1),\n", "\n", " Returns\n", " -------\n", " x: torch.Tensor\n", " (B, SEQ_LEN, OUT_FEATURES)\n", " \"\"\"\n", " x = super(MaskedLinear, self).forward(x)\n", " if mask is not None:\n", " x = x.masked_fill(mask, float(self.mask_value))\n", " return x\n", "\n", " def extra_repr(self):\n", " return (\n", " f\"in_features={self.in_features}, out_features={self.out_features}, \"\n", " f\"mask_value={self.mask_value}, bias={self.bias is not None}\"\n", " )\n", "\n", "\n", "class TilesMLP(torch.nn.Module):\n", " \"\"\"MLP to be applied to tiles to compute scores.\n", " This module can be used in combination of a mask\n", " to prevent padding from influencing the scores values.\n", " Parameters\n", " ----------\n", " in_features: int\n", " size of each input sample\n", " out_features: int\n", " size of each output sample\n", " hidden: Optional[List[int]] = None\n", " Number of hidden layers and their respective number of features.\n", " bias: bool = True\n", " If set to ``False``, the layer will not learn an additive bias.\n", " activation: torch.nn.Module = torch.nn.Sigmoid()\n", " MLP activation function\n", " dropout: Optional[torch.nn.Module] = None\n", " Optional dropout module. Will be interlaced with the linear layers.\n", " \"\"\"\n", "\n", " def __init__(\n", " self,\n", " in_features: int,\n", " out_features: int = 1,\n", " hidden: Optional[List[int]] = None,\n", " bias: bool = True,\n", " activation: torch.nn.Module = torch.nn.Sigmoid(),\n", " dropout: Optional[torch.nn.Module] = None,\n", " ):\n", " super(TilesMLP, self).__init__()\n", "\n", " self.hidden_layers = torch.nn.ModuleList()\n", " if hidden is not None:\n", " for h in hidden:\n", " self.hidden_layers.append(\n", " MaskedLinear(in_features, h, bias=bias, mask_value=\"-inf\")\n", " )\n", " self.hidden_layers.append(activation)\n", " if dropout:\n", " self.hidden_layers.append(dropout)\n", " in_features = h\n", "\n", " self.hidden_layers.append(\n", " torch.nn.Linear(in_features, out_features, bias=bias)\n", " )\n", "\n", " def forward(\n", " self, x: torch.Tensor, mask: Optional[torch.BoolTensor] = None\n", " ):\n", " \"\"\"Forward pass.\n", "\n", " Parameters\n", " ----------\n", " x: torch.Tensor\n", " (B, N_TILES, IN_FEATURES)\n", " mask: Optional[torch.BoolTensor] = None\n", " (B, N_TILES), True for values that were padded.\n", "\n", " Returns\n", " -------\n", " x: torch.Tensor\n", " (B, N_TILES, OUT_FEATURES)\n", " \"\"\"\n", " for layer in self.hidden_layers:\n", " # print(x.size())\n", " if isinstance(layer, MaskedLinear):\n", " x = layer(x, mask)\n", " else:\n", " x = layer(x)\n", "\n", " return x\n", "\n", "\n", "class ExtremeLayer(torch.nn.Module):\n", " \"\"\"Extreme layer.\n", " Returns concatenation of n_top top tiles and n_bottom bottom tiles\n", " .. warning::\n", " If top tiles or bottom tiles is superior to the true number of\n", " tiles in the input then padded tiles will be selected and their value\n", " will be 0.\n", " Parameters\n", " ----------\n", " n_top: Optional[int] = None\n", " Number of top tiles to select\n", " n_bottom: Optional[int] = None\n", " Number of bottom tiles to select\n", " dim: int = 1\n", " Dimension to select top/bottom tiles from\n", " return_indices: bool = False\n", " Whether to return the indices of the extreme tiles\n", "\n", " Raises\n", " ------\n", " ValueError\n", " If ``n_top`` and ``n_bottom`` are set to ``None`` or both are 0.\n", " \"\"\"\n", "\n", " def __init__(\n", " self,\n", " n_top: Optional[int] = None,\n", " n_bottom: Optional[int] = None,\n", " dim: int = 1,\n", " return_indices: bool = False,\n", " ):\n", " super(ExtremeLayer, self).__init__()\n", "\n", " if not (n_top is not None or n_bottom is not None):\n", " raise ValueError(\"one of n_top or n_bottom must have a value.\")\n", "\n", " if not (\n", " (n_top is not None and n_top > 0)\n", " or (n_bottom is not None and n_bottom > 0)\n", " ):\n", " raise ValueError(\"one of n_top or n_bottom must have a value > 0.\")\n", "\n", " self.n_top = n_top\n", " self.n_bottom = n_bottom\n", " self.dim = dim\n", " self.return_indices = return_indices\n", "\n", " def forward(\n", " self, x: torch.Tensor, mask: Optional[torch.BoolTensor] = None\n", " ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:\n", " \"\"\"Forward pass.\n", " Parameters\n", " ----------\n", " x: torch.Tensor\n", " Input tensor, shape (B, N_TILES, IN_FEATURES).\n", " mask: Optional[torch.BoolTensor]\n", " True for values that were padded, shape (B, N_TILES, 1).\n", "\n", " Warnings\n", " --------\n", " If top tiles or bottom tiles is superior to the true number of tiles in\n", " the input then padded tiles will be selected and their value will be 0.\n", "\n", " Returns\n", " -------\n", " values: torch.Tensor\n", " Extreme tiles, shape (B, N_TOP + N_BOTTOM).\n", " indices: torch.Tensor\n", " If ``self.return_indices=True``, return extreme tiles' indices.\n", " \"\"\"\n", "\n", " if (\n", " self.n_top\n", " and self.n_bottom\n", " and ((self.n_top + self.n_bottom) > x.shape[self.dim])\n", " ):\n", " warnings.warn(\n", " f\"Sum of tops is larger than the input tensor shape for dimension {self.dim}: \"\n", " f\"{self.n_top + self.n_bottom} > {x.shape[self.dim]}. \"\n", " f\"Values will appear twice (in top and in bottom)\"\n", " )\n", "\n", " top, bottom = None, None\n", " top_idx, bottom_idx = None, None\n", " if mask is not None:\n", " if self.n_top:\n", " top, top_idx = x.masked_fill(mask, float(\"-inf\")).topk(\n", " k=self.n_top, sorted=True, dim=self.dim\n", " )\n", " top_mask = top.eq(float(\"-inf\"))\n", " if top_mask.any():\n", " warnings.warn(\n", " \"The top tiles contain masked values, they will be set to zero.\"\n", " )\n", " top[top_mask] = 0\n", "\n", " if self.n_bottom:\n", " bottom, bottom_idx = x.masked_fill(mask, float(\"inf\")).topk(\n", " k=self.n_bottom, largest=False, sorted=True, dim=self.dim\n", " )\n", " bottom_mask = bottom.eq(float(\"inf\"))\n", " if bottom_mask.any():\n", " warnings.warn(\n", " \"The bottom tiles contain masked values, they will be set to zero.\"\n", " )\n", " bottom[bottom_mask] = 0\n", " else:\n", " if self.n_top:\n", " top, top_idx = x.topk(k=self.n_top, sorted=True, dim=self.dim)\n", " if self.n_bottom:\n", " bottom, bottom_idx = x.topk(\n", " k=self.n_bottom, largest=False, sorted=True, dim=self.dim\n", " )\n", "\n", " if top is not None and bottom is not None:\n", " values = torch.cat([top, bottom], dim=self.dim)\n", " indices = torch.cat([top_idx, bottom_idx], dim=self.dim)\n", " elif top is not None:\n", " values = top\n", " indices = top_idx\n", " elif bottom is not None:\n", " values = bottom\n", " indices = bottom_idx\n", " else:\n", " raise ValueError\n", "\n", " if self.return_indices:\n", " return values, indices\n", " else:\n", " return values\n", "\n", " def extra_repr(self) -> str:\n", " \"\"\"Format representation.\"\"\"\n", " return f\"n_top={self.n_top}, n_bottom={self.n_bottom}\"\n", "\n", "\n", "class Chowder(nn.Module):\n", " \"\"\"Chowder MIL model (See [1]_).\n", "\n", " Example:\n", " >>> module = Chowder(in_features=128, out_features=1, n_top=5, n_bottom=5)\n", " >>> logits, extreme_scores = module(slide, mask=mask)\n", " >>> scores = module.score_model(slide, mask=mask)\n", "\n", " Parameters\n", " ----------\n", " in_features: int\n", " Features (model input) dimension.\n", " out_features: int\n", " Controls the number of scores and, by extension, the number of out_features.\n", " n_top: int\n", " Number of tiles with hightest scores that are selected and fed to the MLP.\n", " n_bottom: int\n", " Number of tiles with lowest scores that are selected and fed to the MLP.\n", " tiles_mlp_hidden: Optional[List[int]] = None\n", " Number of units for layers in the first MLP applied tile wise to compute\n", " a score for each tiles from the tile features.\n", " If `None`, a linear layer is used to compute tile scores.\n", " If e.g. `[128, 64]`, the tile scores are computed with a MLP of dimension\n", " features_dim -> 128 -> 64 -> 1.\n", " mlp_hidden: Optional[List[int]] = None\n", " Number of units for layers of the second MLP that combine top and bottom\n", " scores and outputs a final prediction at the slide-level. If `None`, a\n", " linear layer is used to compute the prediction from the extreme scores.\n", " If e.g. `[128, 64]`, the prediction is computed\n", " with a MLP n_top + n_bottom -> 128 -> 64 -> 1.\n", " mlp_dropout: Optional[List[float]] = None\n", " Dropout that is used for each layer of the MLP. If `None`, no dropout\n", " is used.\n", " mlp_activation: Optional[torch.nn.Module] = torch.nn.Sigmoid\n", " Activation that is used after each layer of the MLP.\n", " bias: bool = True\n", " Whether to add bias for layers of the tiles MLP.\n", "\n", " References\n", " ----------\n", " .. [1] Pierre Courtiol, Eric W. Tramel, Marc Sanselme, and Gilles Wainrib. Classification\n", " and disease localization in histopathology using only global labels: A weakly-supervised\n", " approach. CoRR, abs/1802.02212, 2018.\n", " \"\"\"\n", "\n", " def __init__(\n", " self,\n", " in_features: int,\n", " out_features: int,\n", " n_top: Optional[int] = None,\n", " n_bottom: Optional[int] = None,\n", " tiles_mlp_hidden: Optional[List[int]] = None,\n", " mlp_hidden: Optional[List[int]] = None,\n", " mlp_dropout: Optional[List[float]] = None,\n", " mlp_activation: Optional[torch.nn.Module] = torch.nn.Sigmoid(),\n", " bias: bool = True,\n", " ) -> None:\n", " super(Chowder, self).__init__()\n", " if n_top is None and n_bottom is None:\n", " raise ValueError(\n", " \"At least one of `n_top` or `n_bottom` must not be None.\"\n", " )\n", "\n", " if mlp_dropout is not None:\n", " if mlp_hidden is not None:\n", " assert len(mlp_hidden) == len(\n", " mlp_dropout\n", " ), \"mlp_hidden and mlp_dropout must have the same length\"\n", " else:\n", " raise ValueError(\n", " \"mlp_hidden must have a value and have the same length as mlp_dropout if mlp_dropout is given.\"\n", " )\n", "\n", " self.score_model = TilesMLP(\n", " in_features,\n", " hidden=tiles_mlp_hidden,\n", " bias=bias,\n", " out_features=out_features,\n", " )\n", " self.score_model.apply(self.weight_initialization)\n", "\n", " self.extreme_layer = ExtremeLayer(n_top=n_top, n_bottom=n_bottom)\n", "\n", " mlp_in_features = n_top + n_bottom\n", " self.mlp = MLP(\n", " mlp_in_features,\n", " 1,\n", " hidden=mlp_hidden,\n", " dropout=mlp_dropout,\n", " activation=mlp_activation,\n", " )\n", " self.mlp.apply(self.weight_initialization)\n", "\n", " @staticmethod\n", " def weight_initialization(module: torch.nn.Module) -> None:\n", " \"\"\"Initialize weights for the module using Xavier initialization method,\n", " \"Understanding the difficulty of training deep feedforward neural networks\",\n", " Glorot, X. & Bengio, Y. (2010).\"\"\"\n", " if isinstance(module, torch.nn.Linear):\n", " torch.nn.init.xavier_uniform_(module.weight)\n", "\n", " if module.bias is not None:\n", " module.bias.data.fill_(0.0)\n", "\n", " def forward(\n", " self, features: torch.Tensor, mask: Optional[torch.BoolTensor] = None\n", " ) -> torch.Tensor:\n", " \"\"\"\n", " Parameters\n", " ----------\n", " features: torch.Tensor\n", " (B, N_TILES, IN_FEATURES)\n", " mask: Optional[torch.BoolTensor] = None\n", " (B, N_TILES, 1), True for values that were padded.\n", "\n", " Returns\n", " -------\n", " logits, extreme_scores: Tuple[torch.Tensor, torch.Tensor]:\n", " (B, OUT_FEATURES), (B, N_TOP + N_BOTTOM, OUT_FEATURES)\n", " \"\"\"\n", " scores = self.score_model(x=features, mask=mask)\n", " extreme_scores = self.extreme_layer(\n", " x=scores, mask=mask\n", " ) # (B, N_TOP + N_BOTTOM, OUT_FEATURES)\n", "# print(extreme_scores.size())\n", " # Apply MLP to the N_TOP + N_BOTTOM scores.\n", " y = self.mlp(extreme_scores.transpose(1, 2)) # (B, OUT_FEATURES, 1)\n", "\n", " return y.squeeze(2)\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "b22e5d94", "metadata": { "execution": { "iopub.execute_input": "2023-12-13T04:03:00.198946Z", "iopub.status.busy": "2023-12-13T04:03:00.198040Z", "iopub.status.idle": "2023-12-13T04:03:45.495504Z", "shell.execute_reply": "2023-12-13T04:03:45.494347Z" }, "papermill": { "duration": 45.303791, "end_time": "2023-12-13T04:03:45.498003", "exception": false, "start_time": "2023-12-13T04:03:00.194212", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "\r\n", "yes: standard output: Broken pipe\r\n", "Looking in links: /kaggle/input/pyvips-install/pyvips/\r\n", "Processing /kaggle/input/pyvips-install/pyvips/pyvips-2.2.1-py2.py3-none-any.whl\r\n", "Requirement already satisfied: cffi>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from pyvips==2.2.1) (1.15.1)\r\n", "Requirement already satisfied: pycparser in /opt/conda/lib/python3.10/site-packages (from cffi>=1.0.0->pyvips==2.2.1) (2.21)\r\n", "Installing collected packages: pyvips\r\n", "Successfully installed pyvips-2.2.1\r\n" ] } ], "source": [ "!yes | sudo dpkg -i /kaggle/input/pyvips-install/libvips/*.deb\n", "!pip install /kaggle/input/pyvips-install/pyvips/pyvips-2.2.1-py2.py3-none-any.whl --find-links=/kaggle/input/pyvips-install/pyvips/ --no-index" ] }, { "cell_type": "code", "execution_count": 3, "id": "4b5f7fdc", "metadata": { "execution": { "iopub.execute_input": "2023-12-13T04:03:45.559120Z", "iopub.status.busy": "2023-12-13T04:03:45.558806Z", "iopub.status.idle": "2023-12-13T04:04:19.406320Z", "shell.execute_reply": "2023-12-13T04:04:19.404953Z" }, "papermill": { "duration": 33.881678, "end_time": "2023-12-13T04:04:19.409382", "exception": false, "start_time": "2023-12-13T04:03:45.527704", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n", " warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n", "Some weights of the model checkpoint at /kaggle/input/modelstatedict/owkin/phikon were not used when initializing ViTModel: ['pooler.dense.weight', 'pooler.dense.bias']\n", "- This IS expected if you are initializing ViTModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing ViTModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n" ] } ], "source": [ "import os\n", "import pyvips\n", "import random\n", "os.environ[\"OPENCV_IO_MAX_IMAGE_PIXELS\"] = pow(2,40).__str__()\n", "import cv2\n", "import rasterio\n", "from rasterio.windows import Window\n", "from tqdm import tqdm\n", "import pandas as pd\n", "import PIL.Image as Image\n", "import numpy as np\n", "import torch\n", "from transformers import AutoImageProcessor, ViTModel\n", "from io import BytesIO\n", "import time\n", "from torch.utils.data import Dataset\n", "from torch.utils.data import DataLoader\n", "from joblib.externals.loky.backend.context import get_context\n", "# torch.multiprocessing.set_start_method('spawn')\n", "\n", "os.environ['VIPS_CONCURRENCY'] = '4'\n", "os.environ['VIPS_DISC_THRESHOLD'] = '15gb'\n", "\n", "Image.MAX_IMAGE_PIXELS = 5000000000\n", "random.seed(42)\n", "\n", "\n", "ROOT_DIR = '/kaggle/input/UBC-OCEAN'\n", "TEST_DIR = '/kaggle/input/UBC-OCEAN/test_thumbnails'\n", "ALT_TEST_DIR = '/kaggle/input/UBC-OCEAN/test_images'\n", "# TEST_DIR = '/kaggle/input/UBC-OCEAN/train_thumbnails'\n", "# ALT_TEST_DIR = '/kaggle/input/UBC-OCEAN/train_images'\n", "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", "# weight_path = \"/kaggle/input/modelstatedict/ckpt_0.870370.pth\"\n", "# weight_path = \"/kaggle/input/modelstatedict/ckpt_0.521739.pth\"\n", "# weight_path = \"/kaggle/input/modelstatedict/ckpt_0.907407.pth\"\n", "\n", "owkin_path = \"/kaggle/input/modelstatedict/owkin/phikon\"\n", "# weight_path_list = [\"ckpt_0.972119.pth\", \"ckpt_0.970260.pth\", \"ckpt_0.966543.pth\", \"ckpt_0.972119_t.pth\",\n", "# \"ckpt_0.979554.pth\", \"ckpt_0.907407.pth\", \"ckpt_0.870370.pth\", \"best_0.918605.pth\"]\n", "weight_path_list = [\"ckpt_0.972119.pth\", \"ckpt_0.970260.pth\", \"ckpt_0.966543.pth\", \"ckpt_0.972119_t.pth\", \n", " \"ckpt_0.979554.pth\",\"ckpt_0.907407.pth\",\"best_0.755814.pth\"]\n", "# weight_path_list = [\"ckpt_0.972119.pth\",\"ckpt_0.970260.pth\",\"ckpt_0.966543.pth\",\"ckpt_0.972119_t.pth\",\n", "# \"ckpt_0.907407.pth\",\"ckpt_0.870370.pth\",\"best_0.918605.pth\",\"best_0.755814.pth\"]\n", "# weight_path_list = [\"/kaggle/input/modelstatedict/ckpt_0.907407.pth\",\n", "# \"/kaggle/input/modelstatedict/ckpt_0.870370.pth\", \n", "# \"/kaggle/input/modelstatedict/best.pth\",\n", "# \"/kaggle/input/modelstatedict/best_0.755814.pth\"]\n", "\n", "\n", "def get_wsl(image_id):\n", " if os.path.exists(f\"{ALT_TEST_DIR}/{image_id}.png\"):\n", " return f\"{ALT_TEST_DIR}/{image_id}.png\"\n", " else:\n", " return f\"{TEST_DIR}/{image_id}_thumbnail.png\"\n", "\n", "\n", "def get_thumbnails(image_id):\n", " if os.path.exists(f\"{TEST_DIR}/{image_id}_thumbnail.png\"):\n", " return f\"{TEST_DIR}/{image_id}_thumbnail.png\"\n", " else:\n", " return f\"{ALT_TEST_DIR}/{image_id}.png\"\n", "\n", "\n", "def tma_bbox(bbox):\n", " minx = 10e9\n", " miny = 10e9\n", " maxx = -1\n", " maxy = -1\n", " for x, y, w, h in bbox:\n", " minx = min(x, minx)\n", " miny = min(y, miny)\n", " maxx = max(x + w, maxx)\n", " maxy = max(y + h, maxy)\n", " bbox = [(minx, miny, maxx - minx, maxy - miny)]\n", " return bbox\n", "\n", "\n", "def changeid2str(preds):\n", " str_list = []\n", " for i in range(preds.shape[0]):\n", " str_list.append(id2name[str(preds[i])])\n", " return str_list\n", "\n", "\n", "def gen_maskbbox(path, id, is_tma):\n", " image = cv2.imread(path)\n", " gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n", " if is_tma:\n", " gray = 255 - gray\n", " ret, binary = cv2.threshold(gray, 60, 255, cv2.THRESH_BINARY)\n", " kernel = np.ones((21, 21), np.uint8)\n", " eroded = cv2.erode(binary, kernel, iterations=1)\n", " dilated = cv2.dilate(eroded, kernel, iterations=1)\n", "\n", " # 形态学开闭操作\n", " opened = cv2.morphologyEx(dilated, cv2.MORPH_OPEN, kernel)\n", " closed = cv2.morphologyEx(opened, cv2.MORPH_CLOSE, kernel)\n", " contours, hierarchy = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", " bbox = []\n", " for contour in contours:\n", " x, y, w, h = cv2.boundingRect(contour)\n", " bbox.append((x, y, w, h))\n", " # if not is_tma:\n", " # cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)\n", " if is_tma:\n", " bbox = tma_bbox(bbox)\n", " # cv2.rectangle(image, (bbox[0][0], bbox[0][1]), (bbox[0][0] + bbox[0][2], bbox[0][1] + bbox[0][3]), (0, 255, 0),\n", " # 2)\n", " # cv2.imwrite(\"./data/rect/%s.png\" % id, image)\n", " return bbox\n", " # cv2.imshow('Result', image)\n", " # cv2.waitKey(0)\n", " # cv2.destroyAllWindows()\n", "\n", "\n", "df = pd.read_csv(\"/kaggle/input/UBC-OCEAN/test.csv\")\n", "# df = pd.read_csv(\"/kaggle/input/UBC-OCEAN/train.csv\")\n", "df[\"file_path\"] = df[\"image_id\"].apply(get_wsl)\n", "df[\"thumbnails\"] = df[\"image_id\"].apply(get_thumbnails)\n", "name2id = {\"CC\": \"0\", \"EC\": \"1\", \"HGSC\": \"2\", \"LGSC\": \"3\", \"MC\": \"4\", \"Other\": \"5\"}\n", "id2name = {v: k for k, v in name2id.items()}\n", "patch_size = 256\n", "stride = patch_size\n", "\n", "\n", "image_processor = AutoImageProcessor.from_pretrained(owkin_path)\n", "model = ViTModel.from_pretrained(owkin_path, add_pooling_layer=False)\n", "model = model.eval().cuda()\n", "\n", "\n", "chowder = Chowder(\n", " in_features=768, # output dimension of Phikon\n", " out_features=5, # dimension of predictions (a probability for class \"1\")\n", " n_top=5, # number of top scores in Chowder (in the image, N is 2)\n", " n_bottom=5, # number of bottom scores in Chowder\n", " mlp_hidden=[200, 100], # MLP hidden layers after the max-min layer\n", " mlp_activation=torch.nn.Sigmoid(), # MLP activation\n", " bias=True # bias for first 1D convolution which computes scores\n", ")\n", "# chowder.load_state_dict(torch.load(weight_path))\n", "chowder = chowder.eval().cuda()\n", "\n", "\n", "\n", "tiles_num = 256\n", "df_sub = pd.DataFrame(columns=[\"image_id\", \"label\"])\n", "\n", "\n", "def get_top_10_percent_indices(data):\n", " # 获取前 10% 大的数据索引\n", " num_elements = len(data)\n", " num_top_elements = int(0.12 * num_elements)\n", "# print(\"num_top_elements\",num_top_elements)\n", " \n", " # 使用 sorted 函数获取从大到小排序的数据索引\n", " sorted_indices = sorted(range(num_elements), key=lambda i: data[i], reverse=True)\n", " \n", " # 截取前 10% 大的数据索引\n", " top_10_percent_indices = sorted_indices[num_top_elements]\n", " \n", " return data[top_10_percent_indices]\n", "\n", "\n", "def process_data(df_data):\n", " preds = {\n", " \"image_id\":[],\n", " \"label\":[],\n", " \"H\":[]\n", " }\n", " for idx, path in tqdm(enumerate(df_data[\"file_path\"]), total=len(df_data[\"file_path\"])):\n", " path_thumbnails = df_data[\"thumbnails\"][idx]\n", " src = pyvips.Image.new_from_file(path)\n", " # src = Image.open(path).convert(\"RGB\")\n", " w, h = src.width, src.height\n", " thumbnails_img = Image.open(path_thumbnails).convert(\"RGB\")\n", " wt, ht = thumbnails_img.size\n", " scale_factor = h / ht\n", " is_tma = (w == wt)\n", " bbox = gen_maskbbox(path_thumbnails, df_data[\"image_id\"][idx], is_tma)\n", " # print(\"bbox ok ! \",len(bbox), scale_factor, \" \", bbox)\n", " tiles = []\n", " for (bx, by, bw, bh) in bbox:\n", " if bw * bh < 10e4:\n", " continue\n", " sx, sy, sw, sh = round(bx * scale_factor), round(by * scale_factor), round(bw * scale_factor), round(\n", " bh * scale_factor)\n", " for j in range(0, (sh - patch_size) // stride + 1):\n", " for i in range(0, (sw - patch_size) // stride + 1):\n", " tiles.append((sx + i * stride, sy + j * stride, 256, 256))\n", " random.shuffle(tiles)\n", " # print(\"shuffle ok ! \", len(tiles))\n", " cnt_max = 0\n", " feats = []\n", " for wds in tiles:\n", " if cnt_max < tiles_num:\n", " wx, wy, ww, wh = wds\n", " if (wx + ww) > (w - 1) or (wy + wh) > (h - 1):\n", " continue\n", " crop_img = src.crop(wx, wy, ww, wh).numpy()\n", " crop_img = Image.fromarray(crop_img)\n", " crop_img_gray = crop_img.convert(\"L\")\n", " crop_img_gray = np.array(crop_img_gray)\n", " mask = (crop_img_gray >= 230)\n", " crop_img_gray[mask] = 0\n", " if np.count_nonzero(crop_img_gray) >= patch_size * patch_size * 0.5:\n", " cnt_max += 1\n", " inputs = image_processor(crop_img, return_tensors=\"pt\")\n", " inputs[\"pixel_values\"] = inputs[\"pixel_values\"].cuda()\n", "\n", " # get the features\n", " with torch.no_grad():\n", " outputs = model(**inputs)\n", " features = outputs.last_hidden_state[:, 0, :] # (1, 768) shape\n", " feats.append(features)\n", " wsl_feat = torch.cat(feats, dim=0)\n", " if wsl_feat.size(0) < tiles_num:\n", " repeat_num = tiles_num // wsl_feat.size(0)\n", " pick_num = tiles_num % wsl_feat.size(0)\n", " picks = random.sample(range(0, wsl_feat.size(0)), pick_num)\n", " wsl_feat = torch.cat([wsl_feat.repeat(repeat_num, 1), wsl_feat[picks, :]], dim=0)\n", " with torch.no_grad():\n", "# outputs = chowder(wsl_feat.unsqueeze(0))\n", " pred = []\n", " for ckpt_path in weight_path_list:\n", " chowder.load_state_dict(torch.load(\"/kaggle/input/modelstatedict/\"+ckpt_path))\n", " pred.append(torch.nn.Softmax(dim=1)(chowder(wsl_feat.unsqueeze(0))))\n", " pred = torch.sum(torch.cat(pred, dim=0),dim=0,keepdim=True)\n", "# _, predicted = torch.max(outputs, 1)\n", " _, predicted = torch.max(pred, 1)\n", " # pd.concat([df_sub, {\"image_id\":df_data[\"image_id\"][idx],\"label\":id2name[str(int(predicted[0]))]}])\n", " # preds.append(predicted.detach().cpu().numpy())\n", " preds[\"image_id\"].append(int(df_data[\"image_id\"][idx]))\n", " H = -torch.sum(pred/len(weight_path_list) * torch.log(pred/len(weight_path_list)))\n", " preds[\"H\"].append(float(H.detach().cpu().numpy()))\n", " preds[\"label\"].append(id2name[str(int(predicted[0].cpu()))])\n", "# if H.detach().cpu().numpy() > 1.1:\n", "# preds[\"label\"].append(\"Other\")\n", "# else:\n", "# preds[\"label\"].append(id2name[str(int(predicted[0].cpu()))])\n", " return pd.DataFrame.from_dict(preds)" ] }, { "cell_type": "code", "execution_count": 4, "id": "a3d025aa", "metadata": { "execution": { "iopub.execute_input": "2023-12-13T04:04:19.477254Z", "iopub.status.busy": "2023-12-13T04:04:19.476474Z", "iopub.status.idle": "2023-12-13T04:05:03.710254Z", "shell.execute_reply": "2023-12-13T04:05:03.709106Z" }, "papermill": { "duration": 44.305833, "end_time": "2023-12-13T04:05:03.751488", "exception": false, "start_time": "2023-12-13T04:04:19.445655", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.10/site-packages/numpy/core/fromnumeric.py:57: FutureWarning: 'DataFrame.swapaxes' is deprecated and will be removed in a future version. Please use 'DataFrame.transpose' instead.\n", " return bound(*args, **kwds)\n", "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n", " warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n", "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n", " warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n", "0it [00:00, ?it/s]\n", "100%|██████████| 1/1 [00:33<00:00, 33.41s/it]\n", "/tmp/ipykernel_26/3869971168.py:14: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " df_sub[\"label\"][mask] = \"Other\"\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
image_idlabel
041HGSC
\n", "
" ], "text/plain": [ " image_id label\n", "0 41 HGSC" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from joblib import Parallel, delayed\n", "\n", "\n", "df_parts = np.array_split(df, 2)\n", "\n", "results = Parallel(n_jobs=2)(delayed(process_data)(part.reset_index(drop=True, inplace=False)) for part in df_parts)\n", "\n", "# display(results)\n", "# results[\"image_id\"] = results[\"image_id\"].astype(df[\"image_id\"].dtype)\n", "# display(results)\n", "df_sub = pd.concat(results, ignore_index=True)\n", "thr = get_top_10_percent_indices(df_sub[\"H\"])\n", "mask = df_sub[\"H\"] > 1.3\n", "df_sub[\"label\"][mask] = \"Other\"\n", "del df_sub[\"H\"]\n", "\n", "df_sub[\"image_id\"] = df_sub[\"image_id\"].astype(df[\"image_id\"].dtype)\n", "df_sub.to_csv(\"submission.csv\", index=False)\n", "display(df_sub)\n" ] } ], "metadata": { "kaggle": { "accelerator": "nvidiaTeslaT4", "dataSources": [ { "databundleVersionId": 6924515, "sourceId": 45867, "sourceType": "competition" }, { "datasetId": 3867331, "sourceId": 7175212, "sourceType": "datasetVersion" }, { "sourceId": 151850757, "sourceType": "kernelVersion" } ], "dockerImageVersionId": 30588, "isGpuEnabled": true, "isInternetEnabled": false, "language": "python", "sourceType": "notebook" }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "papermill": { "default_parameters": {}, "duration": 136.921329, "end_time": "2023-12-13T04:05:06.603019", "environment_variables": {}, "exception": null, "input_path": "__notebook__.ipynb", "output_path": "__notebook__.ipynb", "parameters": {}, "start_time": "2023-12-13T04:02:49.681690", "version": "2.4.0" } }, "nbformat": 4, "nbformat_minor": 5 }