{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Import Library**","metadata":{}},{"cell_type":"code","source":"# Basic Library\nimport os\nimport cv2\nimport wandb\nimport shutil\nimport random\nimport zipfile\nimport numpy as np\nimport pandas as  pd\nfrom tqdm import tqdm\nfrom glob import glob\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\n# Pytorch\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\n\n# Augmentation\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:21.198713Z","iopub.execute_input":"2023-06-04T08:51:21.199075Z","iopub.status.idle":"2023-06-04T08:51:21.206477Z","shell.execute_reply.started":"2023-06-04T08:51:21.199044Z","shell.execute_reply":"2023-06-04T08:51:21.205554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Config**","metadata":{}},{"cell_type":"code","source":"config = {\n    # Seed\n    'seed' : 42,\n    \n    # Dataset & DataLoader Config\n    'batch_size' : 128,\n    'num_workers' : 2,\n    'input_size' : 256,\n    'USE_AMP' : True,\n    'PIN_MEMORY' : True,\n    'TRAIN_SHUFFLE' : True,\n    'VALID_SHUFFLE' : False,\n\n    # Optimizer & Scheduler Config\n    'optimizer' : 'Adam',\n    'optimizer_args':\n      {'lr' : 0.0001},\n    'lr_scheduler' : 'CosineAnnealingWarmRestarts',\n    'lr_scheduler_args':\n      {'T_0' : 4,\n      'T_mult' : 2,\n      'eta_min' : 0.000001},\n\n    # Model Config\n    'model_name' : 'efficientnet_b0',\n    'PRETRAINED' : True,\n\n    # Train Config\n    'num_epochs' : 2,\n    'early_stop_patience' : 4}","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:21.211850Z","iopub.execute_input":"2023-06-04T08:51:21.212967Z","iopub.status.idle":"2023-06-04T08:51:21.220614Z","shell.execute_reply.started":"2023-06-04T08:51:21.212854Z","shell.execute_reply":"2023-06-04T08:51:21.219701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Custom Dataset**\n\n- Unzip our Dataset\n- Make a DataFrame for Training Loop\n- Pytorch Custom Dataset\n- Modify the part '__getitem' Function for your own pandas DataFrame\n- Albumentation for Data Augmentation","metadata":{}},{"cell_type":"code","source":"zip_files = ['test', 'train']\n\nfor zip_file in zip_files:\n    with zipfile.ZipFile(\"../input/dogs-vs-cats-redux-kernels-edition/{}.zip\".format(zip_file),\"r\") as z:\n        z.extractall(\".\")\n        print(\"{} unzipped\".format(zip_file))","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:21.224929Z","iopub.execute_input":"2023-06-04T08:51:21.226015Z","iopub.status.idle":"2023-06-04T08:51:32.723836Z","shell.execute_reply.started":"2023-06-04T08:51:21.225983Z","shell.execute_reply":"2023-06-04T08:51:32.722698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a List Containing Image's Path\ntrain_path = glob('/kaggle/working/train/*')\n\n# Make New DF\ndf = pd.DataFrame(columns=['path', 'label'])\n\n# Paste Data\ndf['path'] = train_path\n\n# Get Label Data from Image Path using Apply & Split\ndf['label'] = df['path'].apply(lambda x : x.split('/')[-1].split('.')[0])\n\nprint('Data Prepared!!')\n\n# Label Encoding for Train\nle = LabelEncoder()\ndf['label'] = le.fit_transform(df['label'])\nprint('Label Encoded!!')\n\n# Visualizing a Sample Image\nsample_path = df['path'][0]\nsample_label = df['label'][0]\nsample = cv2.imread(sample_path)\nsample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)\n\nplt.imshow(sample);\nplt.title(sample_label);\nplt.axis('off');\n\n# Save DataFrame to CSV File for Training\ndf.to_csv('cifar10_csv.csv', index=0)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:32.726168Z","iopub.execute_input":"2023-06-04T08:51:32.726556Z","iopub.status.idle":"2023-06-04T08:51:33.325067Z","shell.execute_reply.started":"2023-06-04T08:51:32.726521Z","shell.execute_reply":"2023-06-04T08:51:33.324281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Test Dataset DataFrame**","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')\n\ntest_df['path'] = test_df['id'].apply(lambda x : '/kaggle/working/test/'+str(x)+'.jpg')\n\ntest_df.to_csv('test_df.csv', index=0)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:33.326093Z","iopub.execute_input":"2023-06-04T08:51:33.326436Z","iopub.status.idle":"2023-06-04T08:51:33.393724Z","shell.execute_reply.started":"2023-06-04T08:51:33.326406Z","shell.execute_reply":"2023-06-04T08:51:33.392887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, df, transforms=None, is_train=True):\n        self.df = df\n        self.is_train = is_train\n        self.transforms=transforms\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img = cv2.imread(self.df.iloc[idx]['path'])\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n\n        if self.is_train:\n            label = self.df.iloc[idx]['label'].astype(np.int8)\n            return img, label\n        else:\n            return img\n\ndef get_transform(input_size):\n    train_transform = A.Compose([\n                                A.Resize(input_size, input_size),\n                                A.HorizontalFlip(p=0.5),\n                                A.ColorJitter(0.4, 0.4, 0.4, 0.4, p=0.5),\n                                A.GaussNoise(var_limit=5. / 255., p=0.3),\n                                A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), max_pixel_value=255.0, always_apply=False, p=1.0),\n                                ToTensorV2()\n                                ])\n    test_transform = A.Compose([\n                                A.Resize(input_size, input_size),\n                                A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), max_pixel_value=255.0, always_apply=False, p=1.0),\n                                ToTensorV2()\n                                ])\n    \n    return train_transform, test_transform","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:33.396392Z","iopub.execute_input":"2023-06-04T08:51:33.396803Z","iopub.status.idle":"2023-06-04T08:51:33.407676Z","shell.execute_reply.started":"2023-06-04T08:51:33.396771Z","shell.execute_reply":"2023-06-04T08:51:33.406635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Optimizer**\n- Choose the optimizer for your pipeline\n- SGD, Adam, AdamW","metadata":{}},{"cell_type":"code","source":"def get_optimizer(optimizer_str: str, **kwargs):\n\n    if optimizer_str == 'SGD':\n\n        optimizer = optim.SGD(**kwargs)\n\n    elif optimizer_str == 'Adam':\n\n        optimizer = optim.Adam(**kwargs)\n    \n    elif optimizer_str == 'AdamW':\n        \n        optimizer = optim.AdamW(**kwargs)\n        \n    return optimizer","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:33.409197Z","iopub.execute_input":"2023-06-04T08:51:33.409821Z","iopub.status.idle":"2023-06-04T08:51:33.420162Z","shell.execute_reply.started":"2023-06-04T08:51:33.409791Z","shell.execute_reply":"2023-06-04T08:51:33.419256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Learning Rate Scheduler**\n- If you wanna schedule your optimizer's learning rate, use this!\n- Cosine Annealing, WarmUpStarts","metadata":{}},{"cell_type":"code","source":"def get_scheduler(scheduler_str, **kwargs):\n    scheduler = None\n    \n    if scheduler_str == 'CosineAnnealingLR':\n\n        scheduler = lr_scheduler.CosineAnnealingLR(**kwargs)\n\n    elif scheduler_str == 'CosineAnnealingWarmRestarts':\n\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(**kwargs)\n\n    return scheduler","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:33.421501Z","iopub.execute_input":"2023-06-04T08:51:33.422028Z","iopub.status.idle":"2023-06-04T08:51:33.429778Z","shell.execute_reply.started":"2023-06-04T08:51:33.421996Z","shell.execute_reply":"2023-06-04T08:51:33.428865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Early Stopper**\n- For Early Stopping your training loop","metadata":{}},{"cell_type":"code","source":"class EarlyStopping:\n    def __init__(self, patience=10, delta=0.0, verbose=True):\n        self.patience = patience\n        self.delta = delta\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.verbose = verbose\n        self.val_loss_min = np.Inf\n\n    def __call__(self, val_loss):\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            if self.counter >= self.patience:\n                self.early_stop = True\n                if self.verbose:\n                    print(f'Early Stopping -> Best Score {self.best_score}!')\n        else:\n            self.best_score = score\n            self.counter = 0\n            self.val_loss_min = val_loss\n            if self.verbose:\n                print(f'Best Score Updated -> {self.best_score}')\n\n        return self.early_stop","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:51:33.431062Z","iopub.execute_input":"2023-06-04T08:51:33.431565Z","iopub.status.idle":"2023-06-04T08:51:33.442143Z","shell.execute_reply.started":"2023-06-04T08:51:33.431528Z","shell.execute_reply":"2023-06-04T08:51:33.441266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Custom Model**","metadata":{}},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self, model_name, in_chans=3, num_classes=2, pretrained=True):\n        super(CustomModel, self).__init__()\n        self.backbone = timm.create_model(model_name, in_chans=in_chans, pretrained=pretrained, num_classes=num_classes)\n        \n    def forward(self, batch):\n        out = self.backbone(batch)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:52:41.191539Z","iopub.execute_input":"2023-06-04T08:52:41.191913Z","iopub.status.idle":"2023-06-04T08:52:41.201545Z","shell.execute_reply.started":"2023-06-04T08:52:41.191877Z","shell.execute_reply":"2023-06-04T08:52:41.200296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Custom Trainer**\n- Using Wandb for Monitoring & Tracking our pipeline","metadata":{}},{"cell_type":"code","source":"class WandbTrainer:\n    def __init__(self, model, optimizer, criterion, device, config, early_stopper=None, lr_scheduler=None):\n        self.model = model\n        self.optimizer = optimizer\n        self.criterion = criterion\n        self.device = device\n        self.config = config\n        self.early_stopper = early_stopper\n        self.scaler = None\n        self.lr_scheduler=lr_scheduler\n\n    def train_epoch(self, data_loader, epoch, scaler=None):\n        self.model.train()\n        train_losses = []\n        correct_predictions = 0\n        total_predictions = 0\n        loop = tqdm(enumerate(data_loader), total=len(data_loader))\n        for i, (inputs, targets) in loop:\n            inputs = inputs.to(self.device)\n            targets = targets.to(self.device, dtype=torch.long)\n\n            # Mixed precision training\n            if scaler is not None:\n                with autocast():\n                    outputs = self.model(inputs)\n                    loss = self.criterion(outputs, targets)\n                scaler.scale(loss).backward()\n                scaler.step(self.optimizer)\n                scaler.update()\n            else:\n                outputs = self.model(inputs)\n                loss = self.criterion(outputs, targets)\n                loss.backward()\n                self.optimizer.step()\n\n            self.optimizer.zero_grad()\n\n            train_losses.append(loss.item())\n\n            # Calculate accuracy\n            predictions = torch.argmax(outputs, dim=1)\n            correct_predictions += torch.sum(predictions == targets)\n            total_predictions += targets.shape[0]\n\n            # Progress Bar Update\n            loop.set_description(f\"Train Epoch [{epoch}/{self.config['num_epochs']}]\")\n            loop.set_postfix(loss=loss.item(), acc=(correct_predictions/total_predictions).item())\n\n        # Calculate epoch train accuracy and loss\n        train_accuracy = correct_predictions / total_predictions\n        train_loss = np.mean(train_losses)\n\n        # Log to WandB\n        wandb.log({'epoch': epoch+1, 'train_loss': train_loss, 'train_accuracy': train_accuracy})\n\n    def validate(self, data_loader, epoch):\n        self.model.eval()\n        val_losses = []\n        correct_predictions = 0\n        total_predictions = 0\n        loop = tqdm(enumerate(data_loader), total=len(data_loader))\n        with torch.no_grad():\n            for i, (inputs, targets) in loop:\n                inputs = inputs.to(self.device)\n                targets = targets.to(self.device, torch.long)\n\n                outputs = self.model(inputs)\n\n                loss = self.criterion(outputs, targets)\n                val_losses.append(loss.item())\n\n                # Calculate accuracy\n                predictions = torch.argmax(outputs, dim=1)\n                correct_predictions += torch.sum(predictions == targets)\n                total_predictions += targets.shape[0]\n\n                # Progress Bar Update\n                loop.set_description(f\"Valid Epoch [{epoch}/{self.config['num_epochs']}]\")\n                loop.set_postfix(loss=loss.item(), acc=(correct_predictions/total_predictions).item())\n\n        # Calculate epoch validation accuracy and loss\n        val_accuracy = correct_predictions / total_predictions\n        val_loss = np.mean(val_losses)\n\n        # Log to WandB\n        wandb.log({'epoch': epoch+1, 'val_loss': val_loss, 'val_accuracy': val_accuracy})\n\n        return val_loss\n\n    def train(self, train_loader, val_loader=None, scaler=None):\n        for epoch in range(self.config['num_epochs']):\n            self.train_epoch(train_loader, epoch, scaler=scaler)\n            if val_loader is not None:\n                val_loss = self.validate(val_loader, epoch)\n                if self.early_stopper is not None:\n                    early_stop = self.early_stopper(val_loss)\n                    if early_stop:\n                        print(f'Early stopping at epoch {epoch+1}')\n                        break\n                    if self.lr_scheduler is not None:\n                        self.lr_scheduler.step(val_loss)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:52:43.861323Z","iopub.execute_input":"2023-06-04T08:52:43.861682Z","iopub.status.idle":"2023-06-04T08:52:43.887280Z","shell.execute_reply.started":"2023-06-04T08:52:43.861653Z","shell.execute_reply":"2023-06-04T08:52:43.885344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Train Loop**","metadata":{}},{"cell_type":"code","source":"def train(df_path):\n    # Set random seed, deterministic\n    torch.cuda.manual_seed(config['seed'])\n    torch.manual_seed(config['seed'])\n    np.random.seed(config['seed'])\n    random.seed(config['seed'])\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n    # Make Directory for Model Saving\n    date = datetime.now().strftime('%Y-%m-%d-%H%M')\n    result_path = f'results/{date}'\n    config['result_path'] = result_path\n    os.makedirs(result_path, exist_ok=True)\n\n    df = pd.read_csv(df_path)\n    train_df, valid_df = train_test_split(df, test_size=0.2, random_state=config['seed'], shuffle=True, stratify=df['label'])\n\n    # Set device\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n\n    # Login WandB\n    wandb.login\n\n    # Initialize WandB\n    run = wandb.init(project='cats_vs_dogs', config=config)\n    artifact = wandb.Artifact('model', type='model')\n\n    # Define Augmentations\n    train_transforms, valid_transforms = get_transform(input_size=config['input_size'])\n\n    # Create data loaders\n    train_dataset = CustomDataset(df=train_df, transforms=train_transforms, is_train=True)\n    valid_dataset = CustomDataset(df=valid_df, transforms=valid_transforms, is_train=True)\n\n    train_loader = DataLoader(train_dataset,\n                            batch_size=config['batch_size'],\n                            shuffle=config['TRAIN_SHUFFLE'],\n                            pin_memory=config['PIN_MEMORY'],\n                            num_workers=config['num_workers'])\n    valid_loader = DataLoader(valid_dataset,\n                            batch_size=config['batch_size'],\n                            shuffle=config['VALID_SHUFFLE'],\n                            pin_memory=config['PIN_MEMORY'],\n                            num_workers=config['num_workers'])\n\n    # Create model and optimizer\n    model = CustomModel(model_name=config['model_name'], pretrained=config['PRETRAINED']).to(device=device)\n\n    # Wandb Tracking Model\n    wandb.watch(model)\n\n    optimizer = get_optimizer(optimizer_str=config['optimizer'], params=model.parameters(), **config['optimizer_args'])\n\n    # Define loss criterion\n    criterion = nn.CrossEntropyLoss()\n\n    # Define Learning Rate Scheduler\n    lr_scheduler = get_scheduler(config['lr_scheduler'], optimizer=optimizer, **config['lr_scheduler_args'])\n\n    # Initialize early stopper\n    early_stopper = EarlyStopping(patience=config['early_stop_patience'])\n\n    # Initialize trainer\n    trainer = WandbTrainer(model, optimizer, criterion, device, config, early_stopper, lr_scheduler=lr_scheduler)\n\n    # Mixed precision setup\n    scaler = GradScaler(enabled=config['USE_AMP'])\n\n    # Train the model\n    trainer.train(train_loader=train_loader, val_loader=valid_loader, scaler=scaler)\n\n    # Save Model\n    model_path = os.path.join(result_path, 'model_state_dict.pt')\n    torch.save(model.state_dict(), model_path)\n    artifact.add_file(model_path)\n    run.log_artifact(artifact)\n    run.join()\n\n    wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:52:44.158128Z","iopub.execute_input":"2023-06-04T08:52:44.158438Z","iopub.status.idle":"2023-06-04T08:52:44.177033Z","shell.execute_reply.started":"2023-06-04T08:52:44.158413Z","shell.execute_reply":"2023-06-04T08:52:44.176156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train!!","metadata":{}},{"cell_type":"code","source":"train('/kaggle/working/cifar10_csv.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:52:44.548887Z","iopub.execute_input":"2023-06-04T08:52:44.549373Z","iopub.status.idle":"2023-06-04T08:58:58.656621Z","shell.execute_reply.started":"2023-06-04T08:52:44.549346Z","shell.execute_reply":"2023-06-04T08:58:58.655600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Prediction**","metadata":{}},{"cell_type":"code","source":"def predict(test_df):\n    # Define hyperparameters\n\n    # Set random seed, deterministic\n    torch.cuda.manual_seed(config['seed'])\n    torch.manual_seed(config['seed'])\n    np.random.seed(config['seed'])\n    random.seed(config['seed'])\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \n    # Set device\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    # Define Augmentations\n    _, test_transforms = get_transform(input_size=config['input_size'])\n\n    # Create DataLoader\n    test_dataset = CustomDataset(df=test_df, transforms=test_transforms, is_train=False)\n    test_loader = DataLoader(test_dataset,\n                             batch_size=config['batch_size'],\n                             pin_memory=config['PIN_MEMORY'],\n                             num_workers=config['num_workers'],\n                             shuffle=False)\n\n    # Create Model & Load Weights\n    model = CustomModel(model_name=config['model_name']).to(device=device)\n    check_point_path = os.path.join(config['result_path'], 'model_state_dict.pt')\n    check_point = torch.load(check_point_path)\n    model.load_state_dict(check_point)\n\n    # Prediction\n    model.eval()\n    predictions = []\n    with torch.no_grad():\n        for imgs in tqdm(test_loader, total=len(test_loader)):\n            imgs = imgs.to(device, dtype=torch.float32)\n            probs = model(imgs)\n            probs = probs.cpu().detach().numpy()\n            predictions += probs.tolist()\n    preds = np.array(predictions).max(axis=1)\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-06-04T08:58:58.659179Z","iopub.execute_input":"2023-06-04T08:58:58.659551Z","iopub.status.idle":"2023-06-04T08:58:58.671319Z","shell.execute_reply.started":"2023-06-04T08:58:58.659511Z","shell.execute_reply":"2023-06-04T08:58:58.670384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save as Submission File","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')\ntest_df = pd.read_csv('/kaggle/working/test_df.csv')\n\npreds = predict(test_df=test_df)\n\nsample_submission['label'] = preds\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T09:01:50.994340Z","iopub.execute_input":"2023-06-04T09:01:50.995289Z","iopub.status.idle":"2023-06-04T09:02:50.926670Z","shell.execute_reply.started":"2023-06-04T09:01:50.995242Z","shell.execute_reply":"2023-06-04T09:02:50.925609Z"},"trusted":true},"execution_count":null,"outputs":[]}]}