{"cells":[{"metadata":{},"cell_type":"markdown","source":"##  Install Dependence"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:09.243869Z","start_time":"2020-09-22T14:36:09.240102Z"},"hide_input":false,"trusted":false},"cell_type":"code","source":"!pip install pytorch_lightning GPUtil > /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import Library"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:11.139803Z","start_time":"2020-09-22T14:36:09.247705Z"},"trusted":false},"cell_type":"code","source":"import os\nimport zipfile\nimport GPUtil\nimport random\nimport pytorch_lightning as pl\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport torch\nimport torchvision\nimport matplotlib.pyplot as plt\n\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import (Dataset, DataLoader)\nfrom torchvision.transforms import (\n        Resize,\n        Compose,\n        ToTensor,\n        Normalize,\n        RandomOrder,\n        ColorJitter,\n        RandomRotation,\n        RandomGrayscale,\n        RandomResizedCrop,\n        RandomVerticalFlip,\n        RandomHorizontalFlip)\n\nfrom PIL import Image, ImageDraw, ImageFont\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_lightning.callbacks import EarlyStopping\nfrom pytorch_lightning.callbacks import ModelCheckpoint\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Show Version"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:11.150258Z","start_time":"2020-09-22T14:36:11.142725Z"},"trusted":false},"cell_type":"code","source":"np.__version__, pd.__version__, sns.__version__","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:11.164550Z","start_time":"2020-09-22T14:36:11.152607Z"},"trusted":false},"cell_type":"code","source":"torch.__version__, torchvision.__version__, pl.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Global Constants"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T15:07:51.736024Z","start_time":"2020-09-22T15:07:51.722914Z"},"trusted":false},"cell_type":"code","source":"RNG_SEED = 9527\nDATA_ROOT = '/kaggle/input/dogs-vs-cats'\nWORK_ROOT = '/kaggle/working'\nCKPT_PATH = f'{WORK_ROOT}/checkpoints/best.ckpt'\nIMGS_ROOT = f'{WORK_ROOT}/temp_unzip'\nSUBMITCSV = f'{WORK_ROOT}/submission.csv'\nFONT_PATH = '/usr/share/fonts/truetype/dejavu/DejaVuSerif-Bold.ttf'\nLABEL_ID_MAP = {'dog': 0, 'cat': 1}\nID_LABEL_MAP = {0: 'dog', 1: 'cat'}\n\nINPUT_SIZE = 224\nBATCH_SIZE = 128\nNUM_CLASSES = 2\n\nMAX_EPOCHS = 3\n\nDATASET_MEAN = (0.485, 0.456, 0.406)\nDATASET_STD = (0.229, 0.224, 0.225)\n\nTEST_SPLIT = 0.3","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T15:07:53.948417Z","start_time":"2020-09-22T15:07:53.229226Z"},"trusted":false},"cell_type":"code","source":"!ls -l $WORK_ROOT","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Random Seed"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T15:07:57.454943Z","start_time":"2020-09-22T15:07:57.445648Z"},"trusted":false},"cell_type":"code","source":"torch.manual_seed(RNG_SEED)\nnp.random.seed(RNG_SEED)\nrandom.seed(RNG_SEED)\n\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare Data"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T15:08:08.855659Z","start_time":"2020-09-22T15:08:00.564655Z"},"trusted":false},"cell_type":"code","source":"if not os.path.exists(f'{IMGS_ROOT}/train'):\n    with zipfile.ZipFile(f'{DATA_ROOT}/train.zip', 'r') as z:\n        z.extractall(f'{IMGS_ROOT}')\n        \nif not os.path.exists(f'{IMGS_ROOT}/test1'):\n    with zipfile.ZipFile(f'{DATA_ROOT}/test1.zip', 'r') as z:\n        z.extractall(f'{IMGS_ROOT}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train Data"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T15:08:11.303563Z","start_time":"2020-09-22T15:08:11.219103Z"},"trusted":false},"cell_type":"code","source":"filenames = os.listdir(f'{IMGS_ROOT}/train')\nlabel_ids = [LABEL_ID_MAP[str(fname)[:3]] for fname in filenames]\ntrain_df = pd.DataFrame({'filename': filenames, 'label': label_ids})\ntrain_df[10:15]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Valid Data"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:11.913294Z","start_time":"2020-09-22T14:36:11.892595Z"},"trusted":false},"cell_type":"code","source":"train_df, valid_df = train_test_split(train_df, test_size = TEST_SPLIT)\nvalid_df[10:15]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test Data"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T15:08:19.037648Z","start_time":"2020-09-22T15:08:19.014626Z"},"trusted":false},"cell_type":"code","source":"filenames = os.listdir(f'{IMGS_ROOT}/test1')\nlabel_ids = [ -1 for x in filenames]\ntest_df = pd.DataFrame({'filename': filenames, 'label': label_ids})\ntest_df[10:15]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sample Data Exploration"},{"metadata":{},"cell_type":"markdown","source":"### Data Distribution"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:12.122230Z","start_time":"2020-09-22T14:36:11.932354Z"},"trusted":false},"cell_type":"code","source":"sns.countplot(x='label',data=train_df).set_title(\"Train Data Distribution\");","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:12.259402Z","start_time":"2020-09-22T14:36:12.124525Z"},"trusted":false},"cell_type":"code","source":"sns.countplot(x='label',data=valid_df).set_title(\"Valid Data Distribution\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define Display Image Method"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:12.271328Z","start_time":"2020-09-22T14:36:12.261848Z"},"trusted":false},"cell_type":"code","source":"def draw_image(filepath, labelname, resize=None, augtrans=None):\n    img = Image.open(filepath).convert('RGB')\n    if resize is not None:\n        img = img.resize((resize, resize))\n    if augtrans is not None:\n        img = augtrans(img)\n        \n    font_obj = ImageFont.truetype(FONT_PATH, 48)\n    draw_img = ImageDraw.Draw(img)\n    font = ImageFont.load_default()\n    draw_img.text((0, 0), labelname, font=font_obj, fill=(0, 0, 255))\n    return np.array(img)\n\ndef grid_image(imgs_list, cols=4):\n    images = torch.as_tensor(imgs_list) # [(W, H, C)...] to (B, H, W, C)\n    images = images.permute(0, 3, 1, 2) # (B, H, W, C) to (B, C, H, W)\n    images = torchvision.utils.make_grid(images, nrow=4) # (C, 2*H, 4*W)\n    images = images.permute(1, 2, 0) # (H, W, C)\n    return images","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display Raw Sample Images"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:14.077308Z","start_time":"2020-09-22T14:36:12.273828Z"},"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(24, 12))\n\nimages_2x4 = [\n    draw_image(\n        filepath=f'{IMGS_ROOT}/train/{row.filename}',\n        labelname=f'{ID_LABEL_MAP[row.label]}',\n        resize=INPUT_SIZE\n    ) for _, row in train_df[:8].iterrows()\n]\n\nplt.imshow(grid_image(images_2x4, cols=4));","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Augment Transform"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:14.087201Z","start_time":"2020-09-22T14:36:14.079837Z"},"trusted":false},"cell_type":"code","source":"aug_trans = RandomOrder([\n    RandomResizedCrop((INPUT_SIZE, INPUT_SIZE)),\n    RandomRotation(degrees=10),\n    RandomVerticalFlip(p=0.3),\n    RandomHorizontalFlip(p=0.3),\n    ColorJitter(brightness=0.55, contrast=0.3, saturation=0.25, hue=0),\n])\n\nimg_trans = Compose([\n    Resize((INPUT_SIZE, INPUT_SIZE)),\n    ToTensor(),\n    Normalize(mean=DATASET_MEAN, std=DATASET_STD),\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display Augment Images"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:15.897050Z","start_time":"2020-09-22T14:36:14.089941Z"},"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(24, 12))\n\ntrans_images_2x4 = [\n    draw_image(\n        filepath=f'{IMGS_ROOT}/train/{row.filename}',\n        labelname=f'{ID_LABEL_MAP[row.label]}',\n        resize=INPUT_SIZE,\n        augtrans = aug_trans\n    ) for _, row in train_df[:8].iterrows()\n]\n\nplt.imshow(grid_image(trans_images_2x4, cols=4));","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Network"},{"metadata":{},"cell_type":"markdown","source":"### Load Pretrained Model"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:16.771450Z","start_time":"2020-09-22T14:36:15.899941Z"},"trusted":false},"cell_type":"code","source":"backbone = torchvision.models.resnet50(pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-21T09:47:38.878375Z","start_time":"2020-09-21T09:47:36.189307Z"}},"cell_type":"markdown","source":"### Freezing Layers"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:16.781307Z","start_time":"2020-09-22T14:36:16.775001Z"},"trusted":false},"cell_type":"code","source":"for param in backbone.parameters():\n    param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Remove FC Layer"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:16.794141Z","start_time":"2020-09-22T14:36:16.784347Z"},"trusted":false},"cell_type":"code","source":"extractor = list(backbone.children())[:-2] # avgpool and fc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model/ Loss/Optimizer "},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:16.861752Z","start_time":"2020-09-22T14:36:16.797272Z"},"trusted":false},"cell_type":"code","source":"METRICS = {\n    'epoch':[],\n    'train_loss':[],\n    'train_acc':[],\n    'val_acc':[],\n    'val_loss':[],\n}\n\nclass DCDataset(Dataset):\n    def __init__(self, root, df, augtrans=None, imgtrans=ToTensor()):\n        super().__init__()\n        self.data = [(f'{root}/{row.filename}', row.label) for _, row in df.iterrows()]\n        self.augtrans = augtrans\n        self.imgtrans = imgtrans\n    \n    def __getitem__(self, index):\n        imgpath, label = self.data[index]\n        img = Image.open(imgpath).convert('RGB')\n        if self.augtrans:\n            img = self.augtrans(img)\n        img = self.imgtrans(img)\n        return img, label, imgpath\n    \n    def __len__(self):\n        return len(self.data)\n\nclass DCNet(pl.LightningModule):\n    def __init__(self, extractor, num_classes=NUM_CLASSES):\n        super().__init__()\n        self.features = nn.Sequential(\n            *extractor, # 2048, 7, 7\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(in_channels=2048, out_channels=1024, kernel_size=3, padding=1),\n            nn.Conv2d(in_channels=1024, out_channels=512, kernel_size=1),\n            nn.Conv2d(in_channels=512, out_channels=256, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.BatchNorm2d(num_features=256, momentum=0.1),\n            nn.MaxPool2d(kernel_size=5, stride=1, padding=2, ceil_mode=False),\n            nn.Dropout(inplace=True, p=0.5)\n        )\n        self.avgpool = nn.AdaptiveAvgPool2d(output_size=(1, 1))\n        self.classifier = nn.Sequential(\n            nn.Flatten(start_dim=1, end_dim=-1),\n            nn.Linear(in_features=256, out_features=128, bias=True),\n            nn.Dropout(inplace=True, p=0.5),\n            nn.Linear(in_features=128, out_features=num_classes, bias=True),\n        )\n\n  \n    def forward(self, x, *args, **kwargs):\n        x = self.features(x)\n        x = self.avgpool(x)\n        x = self.classifier(x)\n        return x\n        \n    def setup(self, stage):\n        torch.cuda.empty_cache()\n\n    def teardown(self, stage):\n        for idx, gpu in enumerate(GPUtil.getGPUs()):\n            allocmem = round(torch.cuda.memory_allocated(idx) / 1024**2, 2)\n            allocmax = round(torch.cuda.max_memory_allocated(idx) / 1024**2, 2)\n            print(f'({stage})\\tGPU-{idx} mem allocated: {allocmem} MB\\t maxmem allocated: {allocmax} MB')\n            \n    @property\n    def metrics(self):\n        return self.metrics\n        \n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(\n            filter(lambda p: p.requires_grad, model.parameters()),\n            lr=0.001,\n            weight_decay=0.001\n        )\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer,\n            mode='min',\n            factor=0.1,\n            patience=3,\n            min_lr=1e-6)\n        return [optimizer], [scheduler]\n    \n    def prepare_data(self):\n        self.train_dataset = DCDataset(f'{IMGS_ROOT}/train', train_df, aug_trans, img_trans) \n        self.valid_dataset = DCDataset(f'{IMGS_ROOT}/train', valid_df, None, img_trans) \n        self.test_dataset = DCDataset(f'{IMGS_ROOT}/test1', test_df, None, img_trans) \n\n    def train_dataloader(self):\n        return DataLoader(\n                self.train_dataset,\n                batch_size=BATCH_SIZE,\n                num_workers=4,\n                drop_last=True,\n                shuffle=True)\n    \n    def training_step(self, batch, batch_idx):\n        x, y_true, path = batch\n        y_pred = self(x)\n        loss = F.cross_entropy(y_pred, y_true, reduction='mean')\n        acc = (torch.argmax(y_pred, dim=1) == y_true).float().mean()\n        return {'loss': loss, 'acc': acc}\n\n    def training_epoch_end(self, outputs):\n        loss = torch.stack([x['loss'] for x in outputs]).mean()\n        acc = torch.stack([x['acc'] for x in outputs]).mean()\n        METRICS['epoch'].append(self.current_epoch)\n        METRICS['train_loss'].append(loss)\n        METRICS['train_acc'].append(acc)\n        return {'progress_bar': {'train_loss': loss, 'train_acc': acc}}\n\n    def val_dataloader(self):\n        return DataLoader(\n            self.valid_dataset,\n            batch_size=BATCH_SIZE,\n            num_workers=4,\n            drop_last=False,\n            shuffle=False)\n    \n    def validation_step(self, batch, batch_idx):\n        x, y_true, path = batch\n        y_pred = self(x)\n        loss = F.cross_entropy(y_pred, y_true, reduction='mean')\n        acc = (torch.argmax(y_pred, dim=1) == y_true).float().mean()\n        return {'val_loss': loss, 'val_acc': acc}\n\n    def validation_epoch_end(self, outputs):\n        loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        acc = torch.stack([x['val_acc'] for x in outputs]).mean()\n        METRICS['val_loss'].append(loss)\n        METRICS['val_acc'].append(acc)\n        return {'progress_bar': {'val_loss': loss, 'val_acc': acc}}\n    \n    def test_dataloader(self):\n        return DataLoader(\n            self.test_dataset,\n            batch_size=BATCH_SIZE,\n            num_workers=4,\n            drop_last=False,\n            shuffle=False)\n    \n    def test_step(self, batch, batch_idx):\n        x, _, path = batch\n        y_pred = torch.argmax(self(x), dim=1).cpu().numpy()\n        log = {'imgid': [os.path.basename(x).split('.')[0] for x in path], 'label': y_pred}\n        return log\n\n    def test_epoch_end(self, outputs):\n        imgid = np.concatenate([x['imgid'] for x in outputs])\n        label = np.concatenate([x['label'] for x in outputs])\n        return {'id': imgid, 'label': label}\n    \nclass DCTrainer(pl.Trainer):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        \n    def save_checkpoint(self, filepath, weights_only: bool = False):\n        return super().save_checkpoint(CKPT_PATH, weights_only)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:36:17.114268Z","start_time":"2020-09-22T14:36:16.866452Z"},"trusted":false},"cell_type":"code","source":"trainer = DCTrainer(\n    max_epochs=MAX_EPOCHS,\n    logger=False,\n    log_gpu_memory='min_max',\n    weights_summary='top',\n    num_sanity_val_steps=0,\n    progress_bar_refresh_rate=1,\n    check_val_every_n_epoch=1,\n    default_root_dir=WORK_ROOT,\n    resume_from_checkpoint=CKPT_PATH if os.path.exists(CKPT_PATH) else None,\n    early_stop_callback=EarlyStopping(monitor='val_loss', patience=7, mode='min'),\n    checkpoint_callback=ModelCheckpoint(monitor='val_loss', period=5, mode='min'),\n    gpus=[0],\n)\n\nmodel = DCNet(extractor)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:48:02.464207Z","start_time":"2020-09-22T14:36:17.116954Z"},"scrolled":false,"trusted":false},"cell_type":"code","source":"trainer.fit(model);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:55:54.605281Z","start_time":"2020-09-22T14:55:45.238663Z"},"trusted":false},"cell_type":"code","source":"result = trainer.test(model, verbose=False, ckpt_path=CKPT_PATH)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:48:12.339059Z","start_time":"2020-09-22T14:48:12.033615Z"},"trusted":false},"cell_type":"code","source":"result_df = pd.DataFrame(data=result[0])\nsns.countplot(x='label',data=result_df).set_title(\"Predict Data Distribution\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot Metrics"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:48:12.848314Z","start_time":"2020-09-22T14:48:12.341874Z"},"trusted":false},"cell_type":"code","source":"num_epoch = len(METRICS['epoch'])\nfig, axs = plt.subplots(1, 2, figsize=(16, 8))\naxs[0].plot(METRICS['epoch'], METRICS['train_acc'])\naxs[0].plot(METRICS['epoch'], METRICS['val_acc'])\naxs[0].set_title('Accuracy')\naxs[0].set_ylabel('Accuracy')\naxs[0].set_xlabel('Epoch')\naxs[0].legend(['train', 'val'], loc='best')\n\naxs[1].plot(METRICS['epoch'], METRICS['train_loss'])\naxs[1].plot(METRICS['epoch'], METRICS['val_loss'])\naxs[1].set_title('Loss')\naxs[1].set_ylabel('Loss')\naxs[1].set_xlabel('Epoch')\naxs[1].legend(['train', 'val'], loc='best');","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T08:26:58.770807Z","start_time":"2020-09-22T08:19:54.498Z"}},"cell_type":"markdown","source":"## Submission "},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:57:52.181708Z","start_time":"2020-09-22T14:57:52.163747Z"},"trusted":false},"cell_type":"code","source":"result_df.to_csv(SUBMITCSV, index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Clean Output"},{"metadata":{"ExecuteTime":{"end_time":"2020-09-22T14:48:13.562940Z","start_time":"2020-09-22T14:48:12.864499Z"},"trusted":false},"cell_type":"code","source":"!rm -rf $IMGS_ROOT\n!ls -l $WORK_ROOT","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}