{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n! pip install -U timm py7zr","metadata":{"papermill":{"duration":18.418411,"end_time":"2022-03-29T14:09:09.627045","exception":false,"start_time":"2022-03-29T14:08:51.208634","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T14:26:03.736487Z","iopub.execute_input":"2022-04-07T14:26:03.73678Z","iopub.status.idle":"2022-04-07T14:26:17.463713Z","shell.execute_reply.started":"2022-04-07T14:26:03.736703Z","shell.execute_reply":"2022-04-07T14:26:17.462846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport math\nimport copy\nimport time\nimport random\n\nfrom pathlib import Path\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\n\nimport py7zr\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler, AdamW\nfrom torch.optim.lr_scheduler import _LRScheduler, StepLR\nfrom torch.utils.data import Dataset, DataLoader, random_split\n\nimport torchmetrics\nimport torchvision.models as models\nfrom torchvision import transforms as T\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Callback, LightningModule, Trainer, LightningDataModule\nfrom pytorch_lightning.loggers import WandbLogger\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\n\nfrom PIL import Image\n\n# For Image Models\nimport timm\nfrom timm.data.transforms_factory import create_transform\nfrom timm.optim import create_optimizer_v2\n# from efficientnet_pytorch import EfficientNet\n# import torchvision.models as models\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"papermill":{"duration":6.08329,"end_time":"2022-03-29T14:09:15.734195","exception":false,"start_time":"2022-03-29T14:09:09.650905","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T14:27:35.302426Z","iopub.execute_input":"2022-04-07T14:27:35.302693Z","iopub.status.idle":"2022-04-07T14:27:35.313947Z","shell.execute_reply.started":"2022-04-07T14:27:35.302664Z","shell.execute_reply":"2022-04-07T14:27:35.312563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Useful Tool to analyze key metrics during and after training\nimport wandb\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    api_key = user_secrets.get_secret(\"wandb\")\n    wandb.login(key=api_key)\nexcept:\n    print('Something Wrong!')","metadata":{"papermill":{"duration":2.34422,"end_time":"2022-03-29T14:10:10.275603","exception":false,"start_time":"2022-03-29T14:10:07.931383","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-05T03:12:32.396013Z","iopub.execute_input":"2022-04-05T03:12:32.397029Z","iopub.status.idle":"2022-04-05T03:12:32.514114Z","shell.execute_reply.started":"2022-04-05T03:12:32.396991Z","shell.execute_reply":"2022-04-05T03:12:32.513377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUT_DIR = Path(\"/\") / \"kaggle\" / \"working\"\nTEMP_DIR = Path(\"/\") / \"kaggle\" / \"temp\"\n\nROOT_DIR = Path('..') / 'input' / 'cifar-10'\n\nTRAIN_ZIP = ROOT_DIR / 'train.7z'\nTEST_ZIP = ROOT_DIR / 'test.7z'\n\nTRAIN_DIR = TEMP_DIR / 'train'\nTEST_DIR = TEMP_DIR / 'test'\n\nTRAIN_CSV = ROOT_DIR / 'trainLabels.csv'\n\nSAMPLE_SUBMISSION_CSV_PATH = ROOT_DIR / \"sampleSubmission.csv\"\n\nENCODER_CLASSES_PATH = OUTPUT_DIR / \"encoder_classes.npy\"\nTEST_CSV_PATH = OUTPUT_DIR / \"test.csv\"\nTRAIN_CSV_ENCODED_PATH = OUTPUT_DIR / \"encoded_train.csv\"\nCHECKPOINTS_DIR = OUTPUT_DIR / \"checkpoints\"\nSUBMISSION_CSV_PATH = OUTPUT_DIR / \"submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-04-07T14:27:52.058639Z","iopub.execute_input":"2022-04-07T14:27:52.059369Z","iopub.status.idle":"2022-04-07T14:27:52.066489Z","shell.execute_reply.started":"2022-04-07T14:27:52.059318Z","shell.execute_reply":"2022-04-07T14:27:52.065533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"archive = py7zr.SevenZipFile(TRAIN_ZIP, mode='r')\narchive.extractall(TEMP_DIR)\narchive.close()\n\narchive = py7zr.SevenZipFile(TEST_ZIP, mode='r')\narchive.extractall(TEMP_DIR)\narchive.close()","metadata":{"papermill":{"duration":52.149266,"end_time":"2022-03-29T14:10:07.907783","exception":false,"start_time":"2022-03-29T14:09:15.758517","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-05T03:12:32.523823Z","iopub.execute_input":"2022-04-05T03:12:32.524361Z","iopub.status.idle":"2022-04-05T03:27:33.162298Z","shell.execute_reply.started":"2022-04-05T03:12:32.524323Z","shell.execute_reply":"2022-04-05T03:27:33.161592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(TRAIN_CSV)\ntrain_df['file_path'] = train_df.id.apply(lambda x: f\"{TRAIN_DIR / str(x)}.png\")\n\nencoder = LabelEncoder()\ntrain_df['encoded_label'] = encoder.fit_transform(train_df['label'])\nnp.save(ENCODER_CLASSES_PATH, encoder.classes_)\n\nskf = StratifiedKFold(n_splits=5)\nfor fold, ( _, val_) in enumerate(skf.split(X=train_df, y=train_df.encoded_label)):\n      train_df.loc[val_ , \"kfold\"] = fold\n\ntrain_df.to_csv(TRAIN_CSV_ENCODED_PATH)","metadata":{"papermill":{"duration":0.10623,"end_time":"2022-03-29T14:10:10.583454","exception":false,"start_time":"2022-03-29T14:10:10.477224","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-05T03:28:58.180446Z","iopub.execute_input":"2022-04-05T03:28:58.180728Z","iopub.status.idle":"2022-04-05T03:28:58.801906Z","shell.execute_reply.started":"2022-04-05T03:28:58.180673Z","shell.execute_reply":"2022-04-05T03:28:58.801174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(SAMPLE_SUBMISSION_CSV_PATH)\ntest_df['file_path'] = test_df.id.apply(lambda x: f\"{TEST_DIR / str(x)}.png\")\ntest_df['encoded_label'] = 0\n\ntest_df.to_csv(TEST_CSV_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T03:28:58.803302Z","iopub.execute_input":"2022-04-05T03:28:58.803535Z","iopub.status.idle":"2022-04-05T03:29:02.214574Z","shell.execute_reply.started":"2022-04-05T03:28:58.803504Z","shell.execute_reply":"2022-04-05T03:29:02.213847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CIFAR(Dataset):\n    def __init__(self, df, transforms=None):\n        super(CIFAR, self).__init__()\n        \n        self.df = df\n        self.file_paths = df['file_path'].values\n        self.labels = df['encoded_label'].values\n        self.transforms = transforms\n            \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_path = self.file_paths[idx]\n        img = Image.open(img_path)\n        \n        label = self.labels[idx]\n        label = torch.tensor(label, dtype=torch.long)\n        \n        if self.transforms:\n            img = self.transforms(img)\n        \n        return img, label","metadata":{"papermill":{"duration":0.042128,"end_time":"2022-03-29T14:10:10.654891","exception":false,"start_time":"2022-03-29T14:10:10.612763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-05T03:29:02.216045Z","iopub.execute_input":"2022-04-05T03:29:02.216472Z","iopub.status.idle":"2022-04-05T03:29:02.223661Z","shell.execute_reply.started":"2022-04-05T03:29:02.216433Z","shell.execute_reply":"2022-04-05T03:29:02.223035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CIFARDM(LightningDataModule):\n    def __init__(\n        self,\n        train_csv_encoded,\n        test_csv,\n        val_fold,\n        image_size, \n        batch_size,\n        num_workers\n    ):\n        super().__init__()\n        \n        self.save_hyperparameters()\n        \n        self.train_df = pd.read_csv(train_csv_encoded)\n        self.test_df = pd.read_csv(test_csv)\n        \n#         self.transform = create_transform(\n#             input_size=(self.hparams.image_size, self.hparams.image_size),\n#             crop_pct=1.0\n#         )\n        \n        self.train_transform = T.Compose([\n                                            T.Resize((self.hparams.image_size, self.hparams.image_size)),\n                                            T.AutoAugment(policy=T.AutoAugmentPolicy.CIFAR10),\n                                            T.ToTensor(),\n                                            T.Normalize(\n                                                mean=[0.4914, 0.4822, 0.4465], \n                                                std=[0.247, 0.243, 0.261],\n                                            )\n                                        ])\n        \n        self.test_transform = T.Compose([\n                                            T.Resize((self.hparams.image_size, self.hparams.image_size)),\n                                            T.ToTensor(),\n                                            T.Normalize(\n                                                mean=[0.4914, 0.4822, 0.4465], \n                                                std=[0.247, 0.243, 0.261],\n                                            )\n                                        ])\n        \n        \n        \n    def setup(self, stage=None):\n        if stage=='fit' or stage is None:\n            train_df = self.train_df[self.train_df.kfold != self.hparams.val_fold].reset_index(drop=True)\n            val_df = self.train_df[self.train_df.kfold == self.hparams.val_fold].reset_index(drop=True)\n\n            self.train_dataset = CIFAR(train_df, transforms=self.train_transform)\n            self.val_dataset = CIFAR(val_df, transforms=self.test_transform)\n            \n        if stage=='test' or stage is None:\n            self.test_dataset = CIFAR(self.test_df, transforms=self.test_transform)\n            \n    def train_dataloader(self):\n        return self._dataloader(self.train_dataset, train=True)\n    \n    def val_dataloader(self):\n        return self._dataloader(self.val_dataset)\n    \n    def test_dataloader(self):\n        return self._dataloader(self.test_dataset)\n    \n    def _dataloader(self, dataset, train=False):\n        return DataLoader(\n            dataset, \n            batch_size=self.hparams.batch_size,\n            shuffle=train,\n            num_workers=self.hparams.batch_size,\n            pin_memory=True,\n            drop_last=train\n        )","metadata":{"execution":{"iopub.status.busy":"2022-04-05T03:29:02.224944Z","iopub.execute_input":"2022-04-05T03:29:02.225669Z","iopub.status.idle":"2022-04-05T03:29:02.242463Z","shell.execute_reply.started":"2022-04-05T03:29:02.225635Z","shell.execute_reply":"2022-04-05T03:29:02.241735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CIFAR_Model(LightningModule):\n    def __init__(\n        self,\n        optimizer,\n        learning_rate,\n        weight_decay,\n        len_train_dl,\n        epochs\n    ):\n        \n        super().__init__()\n        self.save_hyperparameters()\n        \n        self.sizes = [3, 128, 256, 512]\n        self.params = [512, 256, 128, 128]\n        self.conv_blocks1 = [self.conv_block1(in_f, out_f, kernel_size=3, padding=1, dilation=2, bias=False) for in_f, out_f in zip(self.sizes, self.sizes[1:])]\n        self.conv_blocks2 = [self.conv_block1(in_f, out_f, kernel_size=3, padding=1, dilation=2, bias=False) for in_f, out_f in zip(self.sizes, self.sizes[1:])]\n        \n        self.branch1 = nn.Sequential(\n                            *self.conv_blocks1,\n                            self.conv_block2(kernel_size=3, padding=1)\n                        )\n        \n        self.branch2 = nn.Sequential(\n                            *self.conv_blocks2,\n                            self.conv_block2(kernel_size=3, padding=1)\n                        )\n        \n        self.db = [self.dense_block(in_f, out_f, bias=False) for in_f, out_f in zip(self.params, self.params[1:])]\n        self.dense_layers = nn.Sequential(*self.db)\n        \n        self.ll = nn.Linear(128, 10)\n        \n        self.loss_fn = nn.CrossEntropyLoss()\n        self.metrics = torchmetrics.Accuracy()\n        \n    def conv_block1(self, in_f, out_f, *args, **kwargs):\n        return nn.Sequential(\n            nn.Conv2d(in_f, out_f, *args, **kwargs),\n            nn.BatchNorm2d(out_f),\n            nn.MaxPool2d(2),\n            nn.ReLU(),\n            nn.Dropout(0.2)\n        )\n    \n    def conv_block2(self, *args, **kwargs):\n        return nn.Sequential(\n            nn.Conv2d(512, 720, *args, **kwargs),\n            nn.Flatten(),\n            nn.Linear(2880, 512, bias=False),\n            nn.BatchNorm1d(512),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(512, 256, bias=False),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.2)\n        )\n    \n    def dense_block(self, in_f, out_f, *args, **kwargs):\n        return nn.Sequential(\n            nn.Linear(in_f, out_f, *args, **kwargs),\n            nn.BatchNorm1d(out_f),\n            nn.ReLU(),\n            nn.Dropout(0.2)\n        )\n        \n    def forward(self, x):\n        x = torch.cat([self.branch1(x), self.branch2(x)], dim=1)\n        x = self.dense_layers(x)\n        x = self.ll(x)\n        \n        return F.normalize(x)\n    \n    def training_step(self, batch, batch_idx):\n        return self._step(batch, \"train\")\n    \n    def validation_step(self, batch, batch_idx):\n        return self._step(batch, \"val\")\n    \n    def predict_step(self, batch, batch_idx):\n        x, _ = batch\n        output = self(x)\n        return output\n    \n    def _step(self, batch, step):\n        images, labels = batch\n        \n        outputs = self(images)\n        \n        loss = self.loss_fn(outputs, labels)\n        \n        self.metrics(outputs, labels)\n        \n        self.log(f\"{step}_loss\",loss)\n        self.log(f\"{step}_acc\", self.metrics)\n        \n        return loss\n    \n    def configure_optimizers(self):\n        optimizer = create_optimizer_v2(\n            self.parameters(),\n            opt=self.hparams.optimizer, \n            lr=self.hparams.learning_rate,\n            weight_decay=self.hparams.weight_decay,\n        )\n        \n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer,\n            factor=0.6\n        )\n        \n        scheduler = {\"scheduler\": scheduler, \"interval\": \"step\", \"monitor\": train_loss}\n        \n        return [optimizer], [scheduler]","metadata":{"execution":{"iopub.status.busy":"2022-04-05T03:29:02.316184Z","iopub.execute_input":"2022-04-05T03:29:02.316399Z","iopub.status.idle":"2022-04-05T03:29:02.32933Z","shell.execute_reply.started":"2022-04-05T03:29:02.316374Z","shell.execute_reply":"2022-04-05T03:29:02.327745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(\n    train_csv_encoded=str(TRAIN_CSV_ENCODED_PATH),\n    test_csv=str(TEST_CSV_PATH),\n    val_fold=0.0,\n    image_size=32,\n    batch_size=64,\n    num_workers=2,\n    optimizer=\"adam\",\n    learning_rate=3e-4,\n    weight_decay=1e-6,\n    checkpoints_dir=str(CHECKPOINTS_DIR),\n    auto_lr_find=False,\n    auto_scale_batch_size=False,\n    fast_dev_run=False,\n    gpus=1,\n    max_epochs=60,\n    precision=16,\n    stochastic_weight_avg=True\n):\n    pl.seed_everything(41)\n    \n    \n    datamodule = CIFARDM(\n        train_csv_encoded=train_csv_encoded,\n        test_csv=test_csv,\n        val_fold=val_fold,\n        image_size=image_size,\n        batch_size=batch_size,\n        num_workers=num_workers,\n    )\n    \n    datamodule.setup()\n    len_train_dl = len(datamodule.train_dataloader())\n    \n    module = CIFAR_Model(\n        optimizer=optimizer,\n        learning_rate=learning_rate,\n        weight_decay=weight_decay,\n        len_train_dl=len_train_dl,\n        epochs=max_epochs\n    )\n    \n    wandb_logger = WandbLogger(project='CIFAR', \n                           job_type='Train',\n                           anonymous='must')\n    \n    model_checkpoint = ModelCheckpoint(\n        checkpoints_dir,\n        filename=f\"BranchedModel_{image_size}\",\n        monitor=\"train_loss\",\n    )\n    \n    lr_monitor = LearningRateMonitor(logging_interval='step')\n    \n    trainer = pl.Trainer(\n        logger=wandb_logger,\n        auto_lr_find=auto_lr_find,\n        auto_scale_batch_size=auto_scale_batch_size,\n        benchmark=True,\n        callbacks=[model_checkpoint, lr_monitor],\n        deterministic=True,\n        fast_dev_run=fast_dev_run,\n        gpus=gpus,\n        max_epochs=1 if DEBUG else max_epochs,\n        precision=precision,\n        stochastic_weight_avg=stochastic_weight_avg,\n        limit_train_batches=0.1 if DEBUG else 1.0,\n        limit_val_batches=0.1 if DEBUG else 1.0,\n    )\n    \n    trainer.tune(module, datamodule=datamodule)\n\n    trainer.fit(module, datamodule=datamodule)\n    \n    preds = trainer.predict(module, datamodule.test_dataloader())\n    preds = torch.cat(preds, dim=0)\n    preds_idx = torch.argmax(preds, 1).tolist()\n    preds = encoder.inverse_transform(preds_idx)\n    \n    wandb_logger.finalize(\"success\")\n    wandb.finish()\n    \n    return preds","metadata":{"execution":{"iopub.status.busy":"2022-04-05T03:31:13.062563Z","iopub.execute_input":"2022-04-05T03:31:13.062884Z","iopub.status.idle":"2022-04-05T03:31:13.074057Z","shell.execute_reply.started":"2022-04-05T03:31:13.062849Z","shell.execute_reply":"2022-04-05T03:31:13.07342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG=False\n\npreds = train()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T03:31:19.037293Z","iopub.execute_input":"2022-04-05T03:31:19.037979Z","iopub.status.idle":"2022-04-05T03:40:42.103976Z","shell.execute_reply.started":"2022-04-05T03:31:19.037943Z","shell.execute_reply":"2022-04-05T03:40:42.099655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame()\ndf['id'] = test_df['id']\ndf['label'] = preds\n\ndf.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-02T12:37:50.027807Z","iopub.execute_input":"2022-04-02T12:37:50.028082Z","iopub.status.idle":"2022-04-02T12:45:30.533264Z","shell.execute_reply.started":"2022-04-02T12:37:50.028047Z","shell.execute_reply":"2022-04-02T12:45:30.531585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.026965,"end_time":"2022-03-29T15:57:15.069376","exception":false,"start_time":"2022-03-29T15:57:15.042411","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.026274,"end_time":"2022-03-29T15:57:15.122197","exception":false,"start_time":"2022-03-29T15:57:15.095923","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.026389,"end_time":"2022-03-29T15:57:15.175061","exception":false,"start_time":"2022-03-29T15:57:15.148672","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.026531,"end_time":"2022-03-29T15:57:15.228083","exception":false,"start_time":"2022-03-29T15:57:15.201552","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}