{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# task1","metadata":{}},{"cell_type":"markdown","source":"## Define the model with/without shortcut","metadata":{}},{"cell_type":"code","source":"from functools import partial\nfrom typing import Any, Callable, List, Optional, Type, Union\n\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torch.utils.tensorboard import SummaryWriter\n\n\ndef conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d:\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(\n        in_planes,\n        out_planes,\n        kernel_size=3,\n        stride=stride,\n        padding=dilation,\n        groups=groups,\n        bias=False,\n        dilation=dilation,\n    )\n\n\ndef conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d:\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\n\nclass BasicBlock(nn.Module):\n    expansion: int = 1\n\n    def __init__(\n        self,\n        inplanes: int,\n        planes: int,\n        stride: int = 1,\n        downsample: Optional[nn.Module] = None,\n        groups: int = 1,\n        base_width: int = 64,\n        dilation: int = 1,\n        norm_layer: Optional[Callable[..., nn.Module]] = None,\n        shortcut: bool = True,\n    ) -> None:\n        super().__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError(\"BasicBlock only supports groups=1 and base_width=64\")\n        if dilation > 1:\n            raise NotImplementedError(\"Dilation > 1 not supported in BasicBlock\")\n        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n    #     mycode\n        self.shortcut=shortcut\n\n    def forward(self, x: Tensor) -> Tensor:\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        # mycode\n        if self.shortcut:\n            out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass Bottleneck(nn.Module):\n    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)\n    # while original implementation places the stride at the first 1x1 convolution(self.conv1)\n    # according to \"Deep residual learning for image recognition\"https://arxiv.org/abs/1512.03385.\n    # This variant is also known as ResNet V1.5 and improves accuracy according to\n    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.\n\n    expansion: int = 4\n\n    def __init__(\n        self,\n        inplanes: int,\n        planes: int,\n        stride: int = 1,\n        downsample: Optional[nn.Module] = None,\n        groups: int = 1,\n        base_width: int = 64,\n        dilation: int = 1,\n        norm_layer: Optional[Callable[..., nn.Module]] = None,\n    ) -> None:\n        super().__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        width = int(planes * (base_width / 64.0)) * groups\n        # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv1x1(inplanes, width)\n        self.bn1 = norm_layer(width)\n        self.conv2 = conv3x3(width, width, stride, groups, dilation)\n        self.bn2 = norm_layer(width)\n        self.conv3 = conv1x1(width, planes * self.expansion)\n        self.bn3 = norm_layer(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x: Tensor) -> Tensor:\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass MyResNet(nn.Module):\n    def __init__(\n        self,\n        block: Type[Union[BasicBlock, Bottleneck]],\n        layers: List[int],\n        num_classes: int = 1000,\n        zero_init_residual: bool = False,\n        groups: int = 1,\n        width_per_group: int = 64,\n        replace_stride_with_dilation: Optional[List[bool]] = None,\n        norm_layer: Optional[Callable[..., nn.Module]] = None,\n        shortcut: bool = True,\n    ) -> None:\n        super().__init__()\n        # _log_api_usage_once(self)\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\n                \"replace_stride_with_dilation should be None \"\n                f\"or a 3-element tuple, got {replace_stride_with_dilation}\"\n            )\n        self.groups = groups\n        self.base_width = width_per_group\n        self.shortcut=shortcut # mycode\n        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False)\n        self.bn1 = norm_layer(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2, dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2, dilate=replace_stride_with_dilation[2])\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        # Zero-initialize the last BN in each residual branch,\n        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck) and m.bn3.weight is not None:\n                    nn.init.constant_(m.bn3.weight, 0)  # type: ignore[arg-type]\n                elif isinstance(m, BasicBlock) and m.bn2.weight is not None:\n                    nn.init.constant_(m.bn2.weight, 0)  # type: ignore[arg-type]\n\n    def _make_layer(\n        self,\n        block: Type[Union[BasicBlock, Bottleneck]],\n        planes: int,\n        blocks: int,\n        stride: int = 1,\n        dilate: bool = False,\n    ) -> nn.Sequential:\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                conv1x1(self.inplanes, planes * block.expansion, stride),\n                norm_layer(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(\n            block(\n                self.inplanes, planes, stride, downsample, self.groups, self.base_width, previous_dilation, norm_layer,\n                shortcut=self.shortcut # mycode\n            )\n        )\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(\n                block(\n                    self.inplanes,\n                    planes,\n                    groups=self.groups,\n                    base_width=self.base_width,\n                    dilation=self.dilation,\n                    norm_layer=norm_layer,\n                    shortcut=self.shortcut # mycode\n                )\n            )\n\n        return nn.Sequential(*layers)\n\n    def _forward_impl(self, x: Tensor) -> Tensor:\n        # See note [TorchScript super()]\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.fc(x)\n\n        return x\n\n    def forward(self, x: Tensor) -> Tensor:\n        return self._forward_impl(x)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-11-09T13:33:31.496378Z","iopub.execute_input":"2022-11-09T13:33:31.496904Z","iopub.status.idle":"2022-11-09T13:33:33.018045Z","shell.execute_reply.started":"2022-11-09T13:33:31.496776Z","shell.execute_reply":"2022-11-09T13:33:33.016998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## preparatioin","metadata":{}},{"cell_type":"code","source":"root=\"./\"\nuse_shortcut=False\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(\"using device \", device)\n\n# Hyper-parameters\nnum_epochs = 80\nlearning_rate = 0.001\nif use_shortcut:\n    !mkdir ./logs/shortcut/\n    swriter=SummaryWriter(root+\"logs/shortcut/\",comment=f'_shortcut_')\nelse:\n    !mkdir ./logs/no_shortcut/\n    swriter=SummaryWriter(root+\"logs/no_shortcut/\",comment=f'_no_shortcut_')\n# Image preprocessing modules\ntransform = transforms.Compose([\n    transforms.Pad(4),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomCrop(32),\n    transforms.ToTensor()])\n\n# CIFAR-10 dataset\ndata_root=\"/kaggle/input/dataset1/datase1/cifar-10-python/\"\ntrain_dataset = torchvision.datasets.CIFAR10(root=data_root,\n                                                train=True,\n                                                transform=transform,\n                                                download=False)\n\ntest_dataset = torchvision.datasets.CIFAR10(root=data_root,\n                                            train=False,\n                                            transform=transforms.ToTensor())\n\n# Data loader\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                            batch_size=100,\n                                            shuffle=True)\n\ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n                                            batch_size=100,\n                                            shuffle=False)\n\n\n\nmodel = MyResNet(BasicBlock, [2, 2, 2,2],num_classes=10,shortcut=use_shortcut).to(device)\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\nscheduler=ReduceLROnPlateau(optimizer,verbose=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-09T13:38:13.603255Z","iopub.execute_input":"2022-11-09T13:38:13.603665Z","iopub.status.idle":"2022-11-09T13:38:16.139568Z","shell.execute_reply.started":"2022-11-09T13:38:13.603632Z","shell.execute_reply":"2022-11-09T13:38:16.138195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train","metadata":{}},{"cell_type":"code","source":"\n# Train the model\ntotal_step = len(train_loader)\ncurr_lr = learning_rate\nfor epoch in range(num_epochs):\n    train_loss=0\n    model.train()\n    for i, (images, labels) in enumerate(train_loader):\n        images = images.to(device)\n        labels = labels.to(device)\n\n        # Forward pass\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n\n        # Backward and optimize\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        train_loss+=loss.item()\n\n        if (i + 1) % 100 == 0:\n            print(\"Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}\"\n                    .format(epoch + 1, num_epochs, i + 1, total_step, loss.item()))\n    train_loss/=len(train_loader)\n    swriter.add_scalar(tag='Loss/train_loss', scalar_value=train_loss, global_step=epoch+1)\n\n\n    # Test the model\n    test_loss=0\n    model.eval()\n    correct = 0\n    total = 0    \n    with torch.no_grad():\n        for images, labels in test_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            test_loss+=loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n        print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))\n    test_loss/=len(test_loader)\n    scheduler.step(test_loss)\n    swriter.add_scalar(tag='Loss/test_loss', scalar_value=test_loss, global_step=epoch+1)\n    swriter.add_scalar(tag='Accuracy', scalar_value=correct / total, global_step=epoch+1)\n\n# Test the model\nmodel.eval()\nwith torch.no_grad():\n    correct = 0\n    total = 0\n    for images, labels in test_loader:\n        images = images.to(device)\n        labels = labels.to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n\n    print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))\n\n# Save the model checkpoint\ntorch.save(model.state_dict(), f'resnet_{use_shortcut}.ckpt')","metadata":{"execution":{"iopub.status.busy":"2022-11-08T09:46:41.862042Z","iopub.execute_input":"2022-11-08T09:46:41.862413Z","iopub.status.idle":"2022-11-08T10:22:26.055913Z","shell.execute_reply.started":"2022-11-08T09:46:41.862379Z","shell.execute_reply":"2022-11-08T10:22:26.054593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# task2","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torch.utils.tensorboard import SummaryWriter\nfrom torch.utils.data import TensorDataset\nfrom torch.utils.data import DataLoader,Dataset\nimport torchvision\nfrom torchvision import transforms, datasets\nfrom torchvision.models import GoogLeNet\n\n\nimport json\nimport matplotlib.pyplot as plt\nimport os\nimport time\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:32:01.578090Z","iopub.execute_input":"2022-11-12T08:32:01.579241Z","iopub.status.idle":"2022-11-12T08:32:04.253361Z","shell.execute_reply.started":"2022-11-12T08:32:01.579135Z","shell.execute_reply":"2022-11-12T08:32:04.252393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## data inspection","metadata":{}},{"cell_type":"code","source":"for r,d,f in os.walk(\"/kaggle/input/face-mask-detection/face_mask_detection-main/train\"):\n    print(len(f))\nfor r,d,f in os.walk(\"/kaggle/input/face-mask-detection/face_mask_detection-main/validation\"):\n    print(len(f))\nfor r,d,f in os.walk(\"/kaggle/input/face-mask-detection/face_mask_detection-main/test\"):\n    print(len(f))","metadata":{"execution":{"iopub.status.busy":"2022-11-12T04:25:53.905248Z","iopub.execute_input":"2022-11-12T04:25:53.905719Z","iopub.status.idle":"2022-11-12T04:25:54.271951Z","shell.execute_reply.started":"2022-11-12T04:25:53.905683Z","shell.execute_reply":"2022-11-12T04:25:54.270930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## data augmentation\n","metadata":{}},{"cell_type":"code","source":"# importing PIL\nfrom PIL import Image\n\n\nclass AutoAugmentDataset(Dataset):\n    def __init__(self,root,transform):\n        for r,d,f in os.walk(root):\n            self.root=root\n            self.filenames=f\n\n            break\n        self.transform=transform\n        self.data=[Image.open(self.root+'/'+image) for image in self.filenames]\n        self.save_root='/kaggle/working/new2/'\n            \n        \n    def __len__(self):\n        return len(self.filenames)\n    \n    def __getitem__(self,idx):\n        item=self.data[idx]\n        return item\n    \n    def transform_save(self,round):\n        for r in range(round):\n            transformed=[self.transform(d) for d in self.data]\n            for i,img in enumerate(transformed):\n                path=self.save_root+self.filenames[i].split('.')[0]+f\"_trsfm_{r}.png\"\n                print(path)\n                img.save(path)\n        for i,img in enumerate(self.data):\n            path=self.save_root+self.filenames[i]\n            print(path)\n            img.save(path)\n\ntransform_rules=transforms.RandomChoice([\n                transforms.RandomHorizontalFlip(0.7),\n                transforms.RandomRotation(20),\n                transforms.GaussianBlur((3,3), sigma=(0.1, 2.0)),\n                transforms.Compose([transforms.Pad(10),transforms.Resize((128,128))]),\n                transforms.RandomResizedCrop((128,128),scale=(0.8, 1.0)),\n                transforms.RandomAdjustSharpness(1.25),\n                transforms.RandomAdjustSharpness(0.75),\n                transforms.RandomAutocontrast(),\n            ],p=[0.7,0.7,0.3,0.5,0.6,0.2,0.2,0.3])","metadata":{"execution":{"iopub.status.busy":"2022-11-12T04:05:38.988028Z","iopub.execute_input":"2022-11-12T04:05:38.988468Z","iopub.status.idle":"2022-11-12T04:05:39.002673Z","shell.execute_reply.started":"2022-11-12T04:05:38.988434Z","shell.execute_reply":"2022-11-12T04:05:39.001413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/new2/","metadata":{"execution":{"iopub.status.busy":"2022-11-12T04:05:39.719679Z","iopub.execute_input":"2022-11-12T04:05:39.720097Z","iopub.status.idle":"2022-11-12T04:05:40.822913Z","shell.execute_reply.started":"2022-11-12T04:05:39.720063Z","shell.execute_reply":"2022-11-12T04:05:40.821258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = AutoAugmentDataset(\"/kaggle/input/face-mask-detection/face_mask_detection-main/train/without_mask\",transform=transform_rules)\ndataset.transform_save(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train","metadata":{}},{"cell_type":"code","source":"class Trainer(object):\n    def __init__(self):\n        self.input_root=\"/kaggle/input/\"\n        self.output_root=\"/kaggle/working/\"\n        t=time.strftime('%Y-%m-%d-%H-%M-%S')\n            \n        # Device configuration\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        print(\"using device \", self.device)\n\n        # Hyper-parameters\n        self.num_epochs = 100\n        self.learning_rate = 0.001\n        self.batch_size=64\n        self.loss_weight=[0.7,0.2,0.1]\n        hp_str=f\"_{self.batch_size}_{'_'.join([str(f) for f in self.loss_weight])}_\"\n\n        # Image preprocessing modules\n        self.transform =transforms.ToTensor()\n      \n        \n        self.model = GoogLeNet(num_classes=2, aux_logits= True,init_weights=True).to(self.device)\n\n        # Loss and optimizer\n        self.loss_func = nn.CrossEntropyLoss()\n        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)\n        self.scheduler=ReduceLROnPlateau(self.optimizer,patience=5,verbose=True)\n        \n        self.model_save_path=self.output_root+f\"models/{t+hp_str}/\"\n        self.log_path=self.output_root+f\"logs/{t+hp_str}/\"\n        if not os.path.exists(self.model_save_path):\n            os.makedirs(self.model_save_path)\n        if not os.path.exists(self.log_path):\n            os.makedirs(self.log_path)\n        self.swriter=SummaryWriter(self.log_path)\n\n    \n    def load_data(self,input_root,transform,batch_size,aug=False):\n        # dataset\n        if aug:\n            train_dataset = torchvision.datasets.ImageFolder(input_root+\"face-mask-detection-aug/face_mask_detection-main/train\",transform=transform)\n            valid_dataset = torchvision.datasets.ImageFolder(input_root+\"face-mask-detection-aug/face_mask_detection-main/validation\",transform=transform)\n            test_dataset = torchvision.datasets.ImageFolder(input_root+\"face-mask-detection-aug/face_mask_detection-main/test\",transform=transform)\n        else:\n            train_dataset = torchvision.datasets.ImageFolder(input_root+\"face-mask-detection/face_mask_detection-main/train\",transform=transform)\n            valid_dataset = torchvision.datasets.ImageFolder(input_root+\"face-mask-detection/face_mask_detection-main/validation\",transform=transform)\n            test_dataset = torchvision.datasets.ImageFolder(input_root+\"face-mask-detection/face_mask_detection-main/test\",transform=transform)\n \n        # Data loader\n        self.train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                                    batch_size=batch_size,\n                                                    shuffle=True)\n        self.valid_loader = torch.utils.data.DataLoader(dataset=valid_dataset,\n                                                    batch_size=batch_size,\n                                                    shuffle=False)\n        self.test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n                                                    batch_size=batch_size,\n                                                    shuffle=False)\n    \n    def save_model(self,model,path):\n        torch.save(model.state_dict(), path)\n\n    def train1epoch(self):\n        self.model.train()\n        total_loss=0\n        num=0\n        data_loader=self.train_loader\n        for i,(x,y) in enumerate(data_loader):\n            self.optimizer.zero_grad()\n            x=x.to(self.device)        \n            y=y.to(self.device)\n\n            logits,aux_logits2,aux_logits1=self.model(x)\n            loss0=self.loss_func(logits,y)\n            loss1=self.loss_func(aux_logits2,y)\n            loss2=self.loss_func(aux_logits1,y)\n            loss=loss0*self.loss_weight[0] + loss1 *self.loss_weight[1] + loss2 *self.loss_weight[2]\n            loss.backward()\n            self.optimizer.step()\n            total_loss+=loss.item()\n            num+=len(y)\n\n\n        avg_loss=total_loss/num\n        return avg_loss\n\n    def calc_confuseMatrix(self,pred_y,actual_y):\n        TP,FN,FP,TN=0,0,0,0\n        positive=sum(pred_y)\n        negative=len(pred_y)-positive\n        TP_TN=sum(pred_y==actual_y)\n        TP=sum(np.logical_and(pred_y==actual_y,pred_y==1))\n        TN=TP_TN-TP\n        FP=positive-TP\n        FN=negative-TN\n        return TP,FN,FP,TN\n        \n    def calc_GMean(self,pred_y,actual_y):\n        TP,FN,FP,TN=self.calc_confuseMatrix(pred_y,actual_y)\n        return np.sqrt((TP*TN)/((TP+FN)*(TN+FP)))\n\n    def test1epoch(self,mode):\n        self.model.eval()\n        acc = 0.0  # accumulate accurate number / epoch\n        total_loss=0\n        num=0\n        actual_y=[]\n        pred_y=[]\n        \n        if mode==\"valid\":\n            data_loader=self.valid_loader\n        elif mode==\"test\":\n            data_loader=self.test_loader\n        else:\n            raise Exception(\"!!!\")\n            \n        with torch.no_grad():\n            for x, y in data_loader:\n                x=x.to(self.device)        \n                y=y.to(self.device)\n                outputs = self.model(x)  # eval model only have last output layer\n                loss=self.loss_func(outputs,y)\n                predict_y = torch.max(outputs, dim=1)[1]\n                acc += (predict_y == y).sum().item()\n                total_loss+=loss.item()\n                num+=len(y)\n                actual_y.extend(y.cpu().numpy().tolist())\n                pred_y.extend(predict_y.cpu().numpy().tolist())\n\n        acc = acc / num\n        avg_loss=total_loss/num\n        actual_y=np.array(actual_y)\n        pred_y=np.array(pred_y)\n        \n#         print(actual_y)\n#         print(pred_y)\n        gmean=self.calc_GMean(pred_y,actual_y)\n        return acc,avg_loss,gmean\n\n    def train(self,epoch=3,loss_weight=[0.7,0.2,0.1]):\n        best_acc=0\n\n        for e in range(epoch):\n            train_loss=self.train1epoch()\n            self.swriter.add_scalar(\"Loss/train_loss\",train_loss,e)\n            acc,test_loss,gmean=self.test1epoch(\"valid\")\n            self.swriter.add_scalar(\"Loss/test_loss\",test_loss,e)\n            self.swriter.add_scalar(\"Acc\",acc,e)\n            self.swriter.add_scalar(\"GMean\",gmean,e)\n\n            self.scheduler.step(test_loss)\n            if acc > best_acc:\n                best_acc = acc\n                if e>10:\n                    self.save_model(self.model, self.model_save_path+f\"EPOCH_{e}_loss_{test_loss*100}_ACC_{acc}_gmean_{gmean}.pth\")\n            print('[epoch %d] train_loss: %.7f valid_loss: %.7f  valid_accuracy: %.4f valid_gmean %.4f' %\n                  (e + 1,train_loss*100, test_loss*100, acc,gmean))\n            \n    def predict(self):\n        print(\"start inferring\")\n        acc,test_loss,gmean=self.test1epoch(\"test\")\n        print(f\"acc {acc}\")\n        print(f\"test_loss {test_loss}\")\n        print(f\"gmean {gmean}\")","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:43:26.551390Z","iopub.execute_input":"2022-11-12T08:43:26.551998Z","iopub.status.idle":"2022-11-12T08:43:26.598478Z","shell.execute_reply.started":"2022-11-12T08:43:26.551952Z","shell.execute_reply":"2022-11-12T08:43:26.597225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer=Trainer()\ntrainer.load_data(trainer.input_root,trainer.transform,trainer.batch_size)\ntrainer.train(100,loss_weight=[0.7,0.2,0.1])\ntrainer.predict()\n\n'''\n[epoch 100] train_loss: 0.0069437 valid_loss: 0.3874206  valid_accuracy: 0.9647 valid_gmean 0.9244\nacc 0.9389830508474576\ntest_loss 0.004772630558049275\ngmean 0.9179687721631203\n'''","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:33:00.467482Z","iopub.execute_input":"2022-11-12T08:33:00.467863Z","iopub.status.idle":"2022-11-12T08:36:58.836997Z","shell.execute_reply.started":"2022-11-12T08:33:00.467831Z","shell.execute_reply":"2022-11-12T08:36:58.835723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train on augmented data\ntrainer=Trainer()\ntrainer.load_data(trainer.input_root,trainer.transform,trainer.batch_size,True)\ntrainer.train(trainer.num_epochs,loss_weight=[0.7,0.2,0.1])\ntrainer.predict()\n\n\"\"\"\n[epoch 100] train_loss: 0.0027061 valid_loss: 0.1844178  valid_accuracy: 0.9882 valid_gmean 0.9714\nstart inferring\nacc 0.9576271186440678\ntest_loss 0.003229771693379192\ngmean 0.9567769971222251\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:43:33.066834Z","iopub.execute_input":"2022-11-12T08:43:33.067201Z","iopub.status.idle":"2022-11-12T08:49:04.602981Z","shell.execute_reply.started":"2022-11-12T08:43:33.067171Z","shell.execute_reply":"2022-11-12T08:49:04.601780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer=Trainer()\ntrainer.load_data(trainer.input_root,trainer.transform,trainer.batch_size)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:26:56.344057Z","iopub.execute_input":"2022-11-12T08:26:56.344424Z","iopub.status.idle":"2022-11-12T08:26:56.538661Z","shell.execute_reply.started":"2022-11-12T08:26:56.344397Z","shell.execute_reply":"2022-11-12T08:26:56.537387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred=np.array([1,1,1,0])\ny=np.array([1,0,1,0])\nprint(trainer.calc_confuseMatrix(pred,y))\ntrainer.calc_GMean(pred,y)","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:22:49.788799Z","iopub.execute_input":"2022-11-12T08:22:49.789175Z","iopub.status.idle":"2022-11-12T08:22:49.796213Z","shell.execute_reply.started":"2022-11-12T08:22:49.789146Z","shell.execute_reply":"2022-11-12T08:22:49.795328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.test1epoch(\"test\")","metadata":{"execution":{"iopub.status.busy":"2022-11-12T08:26:58.987160Z","iopub.execute_input":"2022-11-12T08:26:58.987694Z","iopub.status.idle":"2022-11-12T08:27:08.093408Z","shell.execute_reply.started":"2022-11-12T08:26:58.987665Z","shell.execute_reply":"2022-11-12T08:27:08.092634Z"},"trusted":true},"execution_count":null,"outputs":[]}]}