{"cells": [{"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "!pip install lightning"}, {"cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": "import os\nimport gc\nimport re\nimport sys\nimport time\nimport copy\nimport random \nimport glob \nimport zipfile\nimport shutil\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\n\nimport lightning.pytorch as pl  \nfrom lightning.pytorch import seed_everything\nfrom lightning.pytorch.callbacks import ModelCheckpoint, EarlyStopping\n\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2   \nfrom PIL import Image\n\nimport transformers\n\nfrom tqdm import tqdm\nfrom sklearn.metrics import log_loss, accuracy_score, roc_auc_score\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)"}, {"cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": "class CFG :\n    debug_one_epoch = True\n    debug_one_fold = False\n    only_infer = False\n    num_workers = 16\n    batch_size = 64\n    num_epochs = 10\n    lr = 1e-3\n    early_stopping_round = 5\n    warmup_prop = 0.1\n    random_seed = 42\n    n_splits = 5\n    model_name = \"resnet18\" # timm \u3067\u4f7f\u3046\u30e2\u30c7\u30eb\u540d\n    pretrained_path = None\n    train_dir = None # \u5b66\u7fd2\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306e\u30d1\u30b9\n    test_dir = None # \u30c6\u30b9\u30c8\u30c7\u30fc\u30bf\u30bb\u30c3\u30c8\u306e\u30d1\u30b9\n    optimizer = torch.optim.AdamW\n    criterion = nn.BCEWithLogitsLoss()\n    scheduler = transformers.get_linear_schedule_with_warmup\n    input_imgsize = 224\n    data_dir = \"../input/dogs-vs-cats-redux-kernels-edition/\"\n    kaggle_working_dir = \"/kaggle/working/\"\n    \ndef seed_torch(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_torch(CFG.random_seed)\n\nif  CFG.debug_one_epoch :\n    CFG.num_epochs = 1\n\nprint('KAGGLE_URL_BASE' in set(os.environ.keys()))\n"}, {"cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": "submission = pd.read_csv(os.path.join(CFG.data_dir, \"sample_submission.csv\"))\nif 'KAGGLE_URL_BASE' in set(os.environ.keys()) :\n    kaggle_train_dir = os.path.join(CFG.kaggle_working_dir, \"train\")\n    # \u3059\u3067\u306b\u89e3\u51cd\u3055\u308c\u3066\u3044\u308b\u5834\u5408\u306f\u89e3\u51cd\u3057\u306a\u3044\n    if not os.path.exists(kaggle_train_dir) :\n        shutil.unpack_archive(os.path.join(CFG.data_dir, \"train.zip\"), CFG.kaggle_working_dir)\n    \n    kaggle_test_dir = os.path.join(CFG.kaggle_working_dir, \"test\")\n    if not os.path.exists(kaggle_test_dir) :\n        shutil.unpack_archive(os.path.join(CFG.data_dir, \"test.zip\"), CFG.kaggle_working_dir)\n        \n    CFG.data_dir = CFG.kaggle_working_dir\n    \nCFG.train_dir = os.path.join(CFG.data_dir, \"train\")\nCFG.test_dir = os.path.join(CFG.data_dir, \"test\")\n\ntrain_list = glob.glob(os.path.join(CFG.data_dir, \"train\", \"*.jpg\"))\ntest_list = glob.glob(os.path.join(CFG.data_dir, \"test\", \"*.jpg\"))"}, {"cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": "train_df = pd.DataFrame(train_list, columns=[\"path\"])\ntrain_df[\"class\"] = train_df[\"path\"].apply(lambda x : x.split(\"/\")[-1].split(\".\")[0])\ntrain_df[\"class\"] = train_df[\"class\"].map({\"dog\" : 1, \"cat\" : 0})\ntest_df = pd.DataFrame(test_list, columns=[\"path\"])\ntest_df[\"class\"] = -1\n# test_df \u306b\u5bfe\u3057\u3066\u306f path \u306e\u6570\u5b57\u304c\u6607\u9806\u3067\u3042\u308b\u3053\u3068\u3092\u4fdd\u8a3c\u3059\u308b\u305f\u3081\u306b id \u3092\u8ffd\u52a0\ntest_df[\"id\"] = test_df[\"path\"].apply(lambda x : int(x.split(\"/\")[-1].split(\".\")[0]))\ntest_df = test_df.sort_values(\"id\").reset_index(drop=True)"}, {"cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": "train_transform = A.Compose([\n    A.Resize(CFG.input_imgsize, CFG.input_imgsize),\n    A.HorizontalFlip(p=0.5), # 50% \u306e\u78ba\u7387\u3067\u6c34\u5e73\u53cd\u8ee2\n    A.Normalize(),\n    ToTensorV2()\n])\ntest_transform = A.Compose([\n    A.Resize(CFG.input_imgsize, CFG.input_imgsize),\n    A.Normalize(),\n    ToTensorV2()\n])"}, {"cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": "class DogsCatsDataset(Dataset) :\n    def __init__(self, df, transform=None) :\n        self.df = df # \u3055\u3063\u304d\u306e pandas dataframe \u3092\u53d7\u3051\u53d6\u308b\n        self.transform = transform # \u753b\u50cf\u306e\u5909\u63db\u51e6\u7406\u3092\u53d7\u3051\u53d6\u308b\n\n    def __len__(self) :\n        return len(self.df)\n    \n    def __getitem__(self, idx) :\n        img = Image.open(self.df.iloc[idx, 0])\n        img = self.transform(image = np.array(img))[\"image\"]\n        label = self.df.iloc[idx, 1].astype(np.float32)\n        return img, label"}, {"cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": "class DogCatModel(nn.Module) :\n    def __init__(self) :\n        super(DogCatModel, self).__init__()\n        self.model = timm.create_model(CFG.model_name, pretrained=True, num_classes=1)\n        \n    def forward(self, x) :\n        return self.model(x)"}, {"cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": "class dog_vs_cats_pl_model(pl.LightningModule) :\n    def __init__(self, model) :\n        super(dog_vs_cats_pl_model, self).__init__()\n        self.model = model\n        self.criterion = CFG.criterion\n        \n    def forward(self, x) :\n        return self.model(x)\n    \n    def training_step(self, batch, batch_idx) :\n        img, label = batch\n        output = self(img)\n        loss = self.criterion(output.squeeze(-1), label)\n        self.log(\"train_loss\", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n        self.log(\"lr\", self.trainer.optimizers[0].param_groups[0][\"lr\"], on_step=True, on_epoch=False, prog_bar=True, logger=True)\n        return loss\n    \n    def validation_step(self, batch, batch_idx) :\n        img, label = batch\n        output = self(img)\n        loss = self.criterion(output.squeeze(-1), label)\n        self.log(\"valid_loss\", loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n        return loss\n    \n    def predict_step(self, batch, batch_idx) :\n        img, _ = batch\n        output = self(img)\n        output = torch.sigmoid(output).cpu().numpy()\n        return output\n        \n    \n    def configure_optimizers(self) :\n        optimizer = CFG.optimizer(self.parameters(), lr=CFG.lr)\n        num_training_steps = len(self.train_dataloader)*CFG.num_epochs\n        num_warmup_steps = int(num_training_steps * CFG.warmup_prop)\n        scheduler = {\n            \"scheduler\" :  transformers.get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps),\n            \"interval\" : \"step\",\n            \"frequency\" : 1\n        }\n        \n        return [optimizer], [scheduler]\n    "}, {"cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": "def run_train_cv_pl(train, test):\n    kf = KFold(n_splits=CFG.n_splits, shuffle=True, random_state=CFG.random_seed)\n    oof = np.zeros((len(train), 1)) \n    predictions =[]\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train)) :\n        print(f\"====================fold : {fold}====================\")\n        train_df = train.iloc[train_idx].reset_index(drop=True)\n        valid_df = train.iloc[valid_idx].reset_index(drop=True)\n        \n        train_dataset = DogsCatsDataset(train_df, transform=train_transform)\n        valid_dataset = DogsCatsDataset(valid_df, transform=test_transform)\n        test_dataset = DogsCatsDataset(test_df, transform=test_transform)\n        \n        train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=CFG.num_workers, drop_last=True, pin_memory=True)\n        valid_loader = DataLoader(valid_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers)\n        test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers)\n        \n        model = DogCatModel()\n        lightning_model = dog_vs_cats_pl_model(model)\n        lightning_model.train_dataloader = train_loader\n        lightning_model.valid_dataloader = valid_loader\n\n        early_stopping = EarlyStopping(\n            monitor=\"valid_loss\",\n            mode=\"min\", \n            patience=CFG.early_stopping_round,\n        )\n        checkpoint = ModelCheckpoint(\n            monitor=\"valid_loss\", \n            mode=\"min\", \n            dirpath=\"checkpoints\", \n            filename=f\"{CFG.model_name}_fold{fold}\", \n            save_top_k=1,\n        )\n        seed_everything(CFG.random_seed)\n        logger = pl.loggers.TensorBoardLogger(\"logs\", name=f\"{CFG.model_name}_fold{fold}\")\n        trainer = pl.Trainer(max_epochs=CFG.num_epochs, accelerator=\"gpu\", precision=16, logger=logger, callbacks=[early_stopping, checkpoint])\n        trainer.fit(lightning_model, train_loader, valid_loader)\n\n        # best_model = DogCatModel()\n        # best_model.load_state_dict(torch.load(os.path.join(\"checkpoints\", f\"{CFG.model_name}_fold{fold}.ckpt\"))[\"state_dict\"])\n        # best_model.eval()\n        \n        valid_preds_list = trainer.predict(lightning_model, valid_loader)\n        valid_preds_arr = np.concatenate(valid_preds_list)\n        oof[valid_idx] = valid_preds_arr\n        \n        test_preds_list = trainer.predict(lightning_model, test_loader)\n        test_preds_arr = np.concatenate(test_preds_list)\n        predictions.append(test_preds_arr)\n        \n        del model, lightning_model, trainer\n        gc.collect()\n        torch.cuda.empty_cache()\n        \n        if CFG.debug_one_fold :\n            break\n    \n    predictions = np.mean(predictions, axis=0)\n    \n    return {\n        \"oof\" : oof,\n        \"predictions\" : predictions\n    }"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "def main() :\n    if CFG.only_infer :\n        test_dataset = DogsCatsDataset(test_df, transform=test_transform)\n        test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers)\n        model = DogCatModel()\n        lightning_model = dog_vs_cats_pl_model(model)\n        predictions = []\n        for fold in range(CFG.n_splits) :\n            model = lightning_model.load_from_checkpoint(f\"checkpoints/{CFG.model_name}_fold{fold}.ckpt\")\n            trainer = pl.Trainer(accelerator=\"gpu\", precision=16, logger=False)\n            predictions.append(trainer.predict(model, test_loader))\n        submission[\"label\"] = np.mean(predictions, axis=0)\n        submission.to_csv(\"submission.csv\", index=False)\n        \n    else :\n        result = run_train_cv_pl(train_df, test_df)\n        oof_preds = result[\"oof\"]\n        predictions = result[\"predictions\"]\n        submission[\"label\"] = predictions\n        submission.to_csv(\"submission.csv\", index=False)\n        train_df[\"oof_preds\"] = oof_preds   \n        train_df.to_csv(\"oof_preds.csv\", index=False)\n        if CFG.debug_one_fold == False :\n            print(f\"oof log loss : {log_loss(train_df['class'], oof_preds)}\")\n        \nif __name__ == \"__main__\" :\n    main()"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12"}}, "nbformat": 4, "nbformat_minor": 2}