{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"},{"sourceId":218451932,"sourceType":"kernelVersion"},{"sourceId":219037702,"sourceType":"kernelVersion"}],"dockerImageVersionId":30840,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%bash\n\nif [ ! -d \"/kaggle/working/train\" ]; then\n    unzip -q /kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip -d /kaggle/working\nfi\n\nif [ ! -d \"/kaggle/working/test\" ]; then\n    unzip -q /kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip -d /kaggle/working\nfi\n","metadata":{"_uuid":"e987ecd6-4968-4759-83ca-c065ea6fac4d","_cell_guid":"ded39027-6d5d-40d7-9121-b59e6d643bda","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.313093Z","iopub.execute_input":"2025-01-24T08:58:54.313460Z","iopub.status.idle":"2025-01-24T08:58:54.324669Z","shell.execute_reply.started":"2025-01-24T08:58:54.313417Z","shell.execute_reply":"2025-01-24T08:58:54.323726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport numpy as np\nimport random\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nimport cv2\nimport timm\nfrom torch.optim.lr_scheduler import _LRScheduler\nimport math\nimport pandas as pd\nimport tqdm\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2   \nfrom sklearn.model_selection import KFold\nimport pprint","metadata":{"_uuid":"0892fb4d-3cb7-4123-9c40-01bb6d677399","_cell_guid":"44b344e0-7ae3-40f3-b04b-b9e69ba0a17d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.325960Z","iopub.execute_input":"2025-01-24T08:58:54.326289Z","iopub.status.idle":"2025-01-24T08:58:54.332512Z","shell.execute_reply.started":"2025-01-24T08:58:54.326259Z","shell.execute_reply":"2025-01-24T08:58:54.331788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    dog = 1\n    cat = 0\n    train_dir = '/kaggle/working/train'\n    test_dir = '/kaggle/working/test'\n    n_fold = 5\n    num_workers = 16\n    pin_memory = True\n    batch_size = 64\n    seed = 2025\n    drop_last = True\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    epochs = 2\n    early_stopping = 3\n    lr = 1e-4\n    optimizer = torch.optim.AdamW\n    warmup_epochs = 0\n    criterion = nn.BCEWithLogitsLoss()\n    size = (224, 224)\n\ncfg = Config()","metadata":{"_uuid":"dfeaf7f4-0cef-479b-a8b0-8aab8dc9258f","_cell_guid":"40f2de41-abca-42ec-9d31-8301828aa9a8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.333685Z","iopub.execute_input":"2025-01-24T08:58:54.333899Z","iopub.status.idle":"2025-01-24T08:58:54.346856Z","shell.execute_reply.started":"2025-01-24T08:58:54.333881Z","shell.execute_reply":"2025-01-24T08:58:54.346175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed=cfg.seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything()","metadata":{"_uuid":"e9041138-27ff-49ba-ab8d-92c7a2775617","_cell_guid":"09d52595-4c8f-4428-994e-7358c5338d1b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.347623Z","iopub.execute_input":"2025-01-24T08:58:54.347813Z","iopub.status.idle":"2025-01-24T08:58:54.366725Z","shell.execute_reply.started":"2025-01-24T08:58:54.347797Z","shell.execute_reply":"2025-01-24T08:58:54.366082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WarmupCosineAnnealingLR(_LRScheduler):\n    def __init__(self, optimizer, warmup_epochs, total_epochs, eta_min=0, last_epoch=-1):\n        self.warmup_epochs = warmup_epochs\n        self.total_epochs = total_epochs\n        self.eta_min = eta_min\n        super(WarmupCosineAnnealingLR, self).__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        if self.last_epoch < self.warmup_epochs:\n            # Warmup phase\n            warmup_lr = [\n                base_lr * (self.last_epoch + 1) / self.warmup_epochs\n                for base_lr in self.base_lrs\n            ]\n            return warmup_lr\n        else:\n            # Cosine Annealing phase\n            cos_anneal_lr = [\n                self.eta_min + (base_lr - self.eta_min) * \n                (1 + math.cos(math.pi * (self.last_epoch - self.warmup_epochs) / \n                              (self.total_epochs - self.warmup_epochs))) / 2\n                for base_lr in self.base_lrs\n            ]\n            return cos_anneal_lr\n\ndef square_pad_and_resize(image, size):\n    h, w, _ = image.shape\n    \n    max_dim = max(h, w)\n    \n    top = (max_dim - h) // 2\n    bottom = max_dim - h - top\n    left = (max_dim - w) // 2\n    right = max_dim - w - left\n    \n    padded_image = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(0, 0, 0))\n    \n    resized_image = cv2.resize(padded_image, (size))\n    return resized_image\n\nclass DC_Dataset(Dataset):\n    def __init__(self, paths, valid=False):\n        super().__init__()\n        self.paths = paths\n        self.valid = valid\n\n        if not self.valid:\n            self.transform = transform = A.Compose([\n                A.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=180, p=0.5),\n                A.HorizontalFlip(p=0.5),\n                A.VerticalFlip(p=0.5),\n                A.RandomBrightnessContrast(p=0.5),\n                A.RGBShift(p=0.5),\n                A.RandomSizedCrop(min_max_height=(cfg.size[0], cfg.size[0]//2), height=cfg.size[0], width=cfg.size[1]),\n                A.Normalize(),\n                ToTensorV2(),\n            ])\n        else:\n            self.transform = transform = A.Compose([\n                A.Normalize(),\n                ToTensorV2(),\n            ])\n\n    def __len__(self):\n        return len(self.paths)\n\n    def __getitem__(self, index):\n        img = cv2.resize(cv2.imread(self.paths[index]), cfg.size)\n        img = self.transform(image=img)['image']\n        return img\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return torch.mean(x.clamp(min=self.eps).pow(self.p), dim=(-1,-2)).pow(1.0 / self.p)\n\n    def __repr__(self):\n        return f\"{self.__class__.__name__}(p={self.p.data.tolist()[0]:.4f}, eps={self.eps})\"\n\nclass Distill_Model(pl.LightningModule):\n    def __init__(self, model_name='convnext_small', teacher_path=None, pretrained=True, num_batch=0, fold=0):\n        super().__init__()\n        \n        self.model = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=1,  # ヘッドを後で定義\n        )\n        if not ('vit' in model_name or 'mobile' in model_name):\n            num_features = self.model.head.in_features\n            self.model.head = nn.Sequential(\n                GeM(),\n                nn.Linear(self.model.head.in_features, 1)\n            )\n        self.teacher = timm.create_model(\n            'convnext_small',\n            pretrained=pretrained,\n            num_classes=1,  # ヘッドを後で定義\n        )\n        num_features = self.teacher.head.in_features\n        self.teacher.head = nn.Sequential(\n            GeM(),\n            nn.Linear(self.teacher.head.in_features, 1)\n        )\n        #teacher_weight = torch.load(teacher_path,map_location='cpu')['state_dict']\n        #for key in list(teacher_weight):\n        #    teacher_weight[key.replace(\"model.\", \"\")] = teacher_weight.pop(key)\n\n        #self.teacher.load_state_dict(teacher_weight)\n        #self.teacher.eval()\n\n        self.fold = fold\n        self.num_batch = num_batch\n        self.criterion = cfg.criterion\n        self.model_name = model_name\n        self.pretrained = pretrained\n\n        self.save_hyperparameters()\n\n    def forward(self, x):\n        return self.model(x).squeeze()\n\n    def training_step(self, batch, batch_idx):\n        img, label = batch\n        output = self(img)\n        label = cfg.distill_ratio*torch.sigmoid(self.teacher(img).squeeze()) + label*(1-cfg.distill_ratio)\n        loss = self.criterion(output, label)\n        self.log('train_loss', loss, prog_bar=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        img, label = batch\n        output = self(img)\n        loss = self.criterion(output, label)\n        pred = torch.sigmoid(output) > 0.5\n        acc = (pred == label).float().mean()\n        if torch.isnan(loss):\n            return None\n\n        self.log('val_loss', loss, prog_bar=True)\n        self.log('val_acc', acc, prog_bar=True)\n        return loss\n\n    def test_step(self, batch, batch_idx):\n        img, label = batch\n        output = self(img)\n        loss = self.criterion(output, label)\n        pred = torch.sigmoid(output) > 0.5\n        acc = (pred == label).float().mean()\n        self.log('test_loss', loss, prog_bar=True)\n        self.log('test_acc', acc, prog_bar=True)\n\n    def configure_optimizers(self):\n        optimizer = cfg.optimizer(self.model.parameters(), lr=cfg.lr)\n        scheduler = WarmupCosineAnnealingLR(optimizer, warmup_epochs=cfg.warmup_epochs*self.num_batch, total_epochs=cfg.epochs*self.num_batch+1)\n        return {\n            \"optimizer\": optimizer,\n            \"lr_scheduler\": {\n                \"scheduler\": scheduler,\n                \"interval\": \"step\",\n                \"frequency\": 1,\n            }\n        }\n\ndef collate(x):\n    return","metadata":{"_uuid":"01d97035-67a9-4d31-8832-2d646b76db8e","_cell_guid":"eadf6691-ca71-4aef-a0c3-d6ace1472704","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.485633Z","iopub.execute_input":"2025-01-24T08:58:54.485853Z","iopub.status.idle":"2025-01-24T08:58:54.506396Z","shell.execute_reply.started":"2025-01-24T08:58:54.485835Z","shell.execute_reply":"2025-01-24T08:58:54.505614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_paths = glob.glob(cfg.test_dir + '/*')\nimage_ids = []\n\nfor path in test_paths:\n    image_ids.append(os.path.basename(path).split('.')[0])","metadata":{"_uuid":"1227126c-bade-4112-8ce2-dc260601e332","_cell_guid":"2a065900-1690-43e5-9ce6-f9cc79b678fe","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.507607Z","iopub.execute_input":"2025-01-24T08:58:54.507908Z","iopub.status.idle":"2025-01-24T08:58:54.559936Z","shell.execute_reply.started":"2025-01-24T08:58:54.507888Z","shell.execute_reply":"2025-01-24T08:58:54.559391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_paths = glob.glob('/kaggle/input/dogs-vs-cats-distillation/lightning_logs/version_*/checkpoints/*.ckpt')\npprint.pprint(model_paths)","metadata":{"_uuid":"e754b455-c402-4e19-827b-308f4b856031","_cell_guid":"afbec3a0-8fd6-4d20-8039-e246b0950128","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.561222Z","iopub.execute_input":"2025-01-24T08:58:54.561447Z","iopub.status.idle":"2025-01-24T08:58:54.579368Z","shell.execute_reply.started":"2025-01-24T08:58:54.561428Z","shell.execute_reply":"2025-01-24T08:58:54.578763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outputs = []","metadata":{"_uuid":"1b3a8122-6122-468b-a388-3b99fcd42780","_cell_guid":"0c870cda-a0c8-451d-be13-5c564174e667","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.580451Z","iopub.execute_input":"2025-01-24T08:58:54.580680Z","iopub.status.idle":"2025-01-24T08:58:54.583869Z","shell.execute_reply.started":"2025-01-24T08:58:54.580663Z","shell.execute_reply":"2025-01-24T08:58:54.583015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dataset = DC_Dataset(test_paths, valid=True)\ntest_loader = DataLoader(\n    test_dataset, batch_size=cfg.batch_size, shuffle=False,\n    num_workers=cfg.num_workers, pin_memory=cfg.pin_memory,\n)","metadata":{"_uuid":"ccb5114b-c3b2-418c-9399-0173e5149327","_cell_guid":"436561b4-6277-4397-a820-a08c67ef7ca8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.584614Z","iopub.execute_input":"2025-01-24T08:58:54.584830Z","iopub.status.idle":"2025-01-24T08:58:54.599308Z","shell.execute_reply.started":"2025-01-24T08:58:54.584813Z","shell.execute_reply":"2025-01-24T08:58:54.598547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for model_path in model_paths:\n    model = Distill_Model.load_from_checkpoint(model_path)\n    model.eval()\n    outputs.append([])\n    with torch.no_grad():\n        for img in tqdm.tqdm(test_loader):\n            output = model(img.to(cfg.device))\n            outputs[-1] += output.tolist()\n","metadata":{"_uuid":"da59a282-d5a1-49d1-ab21-001f5f6c9c92","_cell_guid":"f01088a3-b206-4c52-9e18-89b519193978","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T08:58:54.599994Z","iopub.execute_input":"2025-01-24T08:58:54.600188Z","iopub.status.idle":"2025-01-24T09:08:32.000524Z","shell.execute_reply.started":"2025-01-24T08:58:54.600173Z","shell.execute_reply":"2025-01-24T09:08:31.999411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outputs = torch.tensor(outputs)\noutputs = outputs.mean(dim=0)\noutputs = torch.sigmoid(outputs)","metadata":{"_uuid":"406c45ae-9dbc-4769-8f32-9ecf94015887","_cell_guid":"edd31bea-0181-48e8-8238-3344969dfa84","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T09:08:32.001611Z","iopub.execute_input":"2025-01-24T09:08:32.001886Z","iopub.status.idle":"2025-01-24T09:08:32.020420Z","shell.execute_reply.started":"2025-01-24T09:08:32.001861Z","shell.execute_reply":"2025-01-24T09:08:32.019582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for clip in [0.01, 0.005, 0.015, 0.0125, 0.0025, 0, 0.0075, 0.004]:\n    submission = pd.DataFrame({'id':image_ids, 'label':torch.clamp(outputs, min=clip, max=1-clip).tolist()})\n    submission['id'] = pd.to_numeric(submission['id'])\n    submission = submission.sort_values(by='id')\n    submission.to_csv(f'/kaggle/working/submission-clip={clip}.csv', index=False)","metadata":{"_uuid":"5a8bb0bf-3b0e-4374-a8ac-506417704d1e","_cell_guid":"69a6dd10-c6d6-451a-800b-2c8b42567bd5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T09:08:32.021380Z","iopub.execute_input":"2025-01-24T09:08:32.021668Z","iopub.status.idle":"2025-01-24T09:08:32.311178Z","shell.execute_reply.started":"2025-01-24T09:08:32.021640Z","shell.execute_reply":"2025-01-24T09:08:32.310259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"_uuid":"8231d4ff-9148-41ec-8203-e803e344fbc2","_cell_guid":"3cd790e6-a52b-43a4-bb27-aadc23d891e4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T09:08:32.313236Z","iopub.execute_input":"2025-01-24T09:08:32.313543Z","iopub.status.idle":"2025-01-24T09:08:32.329603Z","shell.execute_reply.started":"2025-01-24T09:08:32.313520Z","shell.execute_reply":"2025-01-24T09:08:32.328862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!rm -r /kaggle/working/train\n!rm -r /kaggle/working/test","metadata":{"_uuid":"c16d9017-92ac-4834-a970-0b3bd287f9e7","_cell_guid":"49b7b670-a85d-4dd8-b3e5-1249e9d5c3d4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-01-24T09:08:32.330418Z","iopub.execute_input":"2025-01-24T09:08:32.330669Z","iopub.status.idle":"2025-01-24T09:08:33.582557Z","shell.execute_reply.started":"2025-01-24T09:08:32.330640Z","shell.execute_reply":"2025-01-24T09:08:33.581376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"bc2dacac-3330-46c2-9cf6-79cbd7685687","_cell_guid":"0716c216-001e-46c6-83f8-3e6260c7fd1c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}