{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":54041,"databundleVersionId":5884478,"sourceType":"competition"},{"sourceId":5843643,"sourceType":"datasetVersion","datasetId":3352760},{"sourceId":132212144,"sourceType":"kernelVersion"},{"sourceId":132420435,"sourceType":"kernelVersion"}],"dockerImageVersionId":30498,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Summary\n\n> Check version 13 for best public lb score\n\n**CV Strategy**\n\n* Stratified 5 folds.\n\n**Experiments that worked out.**\n\n* Adding label smoothing to the cross entropy loss.\n* Adding classifier dropout.\n* Convnextv2 series performs the best.\n* Used ensemble of convnextv2_tiny(CNN based model) and swin(transformer based model)\n\n\n**Experiments that doesn't worked out.**\n\n* Seam Carving (V14)\n* Arcface instead of linear layer as last layer. (V12)\n* GeM Pooling (V7)\n* Class weights in loss function for class imbalance.\n\n\n**Important Links:**\n\n* [Inference Notebook](https://www.kaggle.com/code/himanshunayal/extracting-attributes-from-fashion-images-infer)\n* [Generating Seam carving notebook](https://www.kaggle.com/code/himanshunayal/seam-carving-eafi-test-data)","metadata":{}},{"cell_type":"code","source":"import os \nimport gc\nimport cv2\nimport glob\nimport random\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport timm\nimport torch\nimport torch.nn as nn\nfrom torch.cuda import amp\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:06:09.019541Z","iopub.execute_input":"2024-01-23T08:06:09.019926Z","iopub.status.idle":"2024-01-23T08:06:15.94739Z","shell.execute_reply.started":"2024-01-23T08:06:09.019894Z","shell.execute_reply":"2024-01-23T08:06:15.946352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# timm.list_models()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T07:28:13.97243Z","iopub.execute_input":"2024-01-23T07:28:13.972713Z","iopub.status.idle":"2024-01-23T07:28:13.977009Z","shell.execute_reply.started":"2024-01-23T07:28:13.972688Z","shell.execute_reply":"2024-01-23T07:28:13.976002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configurations","metadata":{}},{"cell_type":"code","source":"class cfg:\n    data_path = \"/kaggle/input/extracting-attributes-from-fashion-images-2/\"\n    sc_path = \"/kaggle/input/seam-carving-eafi-test-data/\"\n    img_size = (384, 384)\n    num_classes = 7\n    bs = 64\n    ls = 0.1\n    cls_drop = 0.1\n    epochs = 5\n    lr = 1e-4\n    wd = 1e-2","metadata":{"execution":{"iopub.status.busy":"2024-01-23T10:57:39.754985Z","iopub.execute_input":"2024-01-23T10:57:39.755351Z","iopub.status.idle":"2024-01-23T10:57:39.76231Z","shell.execute_reply.started":"2024-01-23T10:57:39.755318Z","shell.execute_reply":"2024-01-23T10:57:39.761263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(cfg.data_path+\"train.csv\")\ndisplay(df_train.head())\nprint(f\"Number of Train samples: {df_train.shape[0]}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:06:22.658669Z","iopub.execute_input":"2024-01-23T08:06:22.65936Z","iopub.status.idle":"2024-01-23T08:06:22.70675Z","shell.execute_reply.started":"2024-01-23T08:06:22.659311Z","shell.execute_reply":"2024-01-23T08:06:22.705771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img_dirs = glob.glob(cfg.data_path+\"test/*\")\ncfg.test_img_dirs = test_img_dirs\nprint(f\"Number of Test samples: {len(test_img_dirs)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:06:26.408875Z","iopub.execute_input":"2024-01-23T08:06:26.409275Z","iopub.status.idle":"2024-01-23T08:06:26.663148Z","shell.execute_reply.started":"2024-01-23T08:06:26.409241Z","shell.execute_reply":"2024-01-23T08:06:26.662185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:06:29.543758Z","iopub.execute_input":"2024-01-23T08:06:29.544129Z","iopub.status.idle":"2024-01-23T08:06:29.557926Z","shell.execute_reply.started":"2024-01-23T08:06:29.544102Z","shell.execute_reply":"2024-01-23T08:06:29.556815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample Images","metadata":{}},{"cell_type":"code","source":"def show_images(path):\n    sample_images = []\n    num_rows = 2 \n    num_cols = 4\n    for i in range(0,7):\n        df_subset = df_train[df_train['label']==i] \n        img = cv2.imread(os.path.join(path+\"train\",df_subset.file_name.values[6]))\n        sample_images.append(img)\n\n    fig, axes = plt.subplots(num_rows, num_cols, figsize=(12, 8))\n    axes = axes.flatten()\n\n    fig.set_facecolor(\"lightblue\")\n    \n    for i in range(len(sample_images)):\n        axes[i].imshow(cv2.cvtColor(sample_images[i], cv2.COLOR_BGR2RGB))\n        axes[i].set_title(f\"Label {i}\")\n        axes[i].axis(\"off\")\n\n    if len(sample_images) < num_rows * num_cols:\n        axes[-1].axis(\"off\")\n\n    plt.tight_layout()\n    \nshow_images(cfg.data_path)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:06:33.153879Z","iopub.execute_input":"2024-01-23T08:06:33.154622Z","iopub.status.idle":"2024-01-23T08:06:34.765307Z","shell.execute_reply.started":"2024-01-23T08:06:33.154591Z","shell.execute_reply":"2024-01-23T08:06:34.764392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class FashionDataset(Dataset):\n    def __init__(self, df=None, train=True):\n        super().__init__()\n        self.train=train\n        self.df = df\n        \n        self.sc_imgs = os.listdir(cfg.sc_path)\n        self.img_dirs = [cfg.data_path+f\"train/{f_name}\" for f_name in df.file_name]\n        self.labels = [l for l in df.label]\n\n        self.train_transform = A.Compose([\n            A.OneOf([\n#                 A.VerticalFlip(p=0.5),  \n                A.HorizontalFlip(p=0.5) \n            ]),\n            A.augmentations.transforms.ChannelShuffle(p=0.3),\n            A.augmentations.dropout.coarse_dropout.CoarseDropout(max_holes=7, max_height=32, max_width=32, p=0.25),\n            A.RandomBrightnessContrast(p=0.2, brightness_by_max=False),\n            A.augmentations.geometric.resize.Resize(height=cfg.img_size[1], width=cfg.img_size[0], always_apply=True),\n            ToTensorV2()\n        ])\n        \n        self.test_transform = A.Compose([ \n            A.augmentations.geometric.resize.Resize(height=cfg.img_size[1], width=cfg.img_size[0], always_apply=True),\n            ToTensorV2()\n        ])\n\n\n    def __len__(self):\n        return len(self.df) \n    \n    def __getitem__(self,idx):\n        img_dir = Path(self.img_dirs[idx])\n        if img_dir.stem+\".jpg\" in self.sc_imgs:\n            img_dir = cfg.sc_path+f\"train/{img_dir.stem}.jpg\"\n\n        label = self.labels[idx]\n        \n        img = cv2.imread(str(img_dir))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = img/255.0\n        \n        if self.train:\n            img = self.train_transform(image=img)['image'].float()\n        else:\n            img = self.test_transform(image=img)['image'].float()\n        \n        return img, label \n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T10:57:49.74509Z","iopub.execute_input":"2024-01-23T10:57:49.745726Z","iopub.status.idle":"2024-01-23T10:57:49.757563Z","shell.execute_reply.started":"2024-01-23T10:57:49.745693Z","shell.execute_reply":"2024-01-23T10:57:49.756605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = FashionDataset(df=df_train)\nplt.imshow(dataset[9][0].permute(1,2,0))\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:06:48.753258Z","iopub.execute_input":"2024-01-23T08:06:48.754153Z","iopub.status.idle":"2024-01-23T08:06:49.191398Z","shell.execute_reply.started":"2024-01-23T08:06:48.754113Z","shell.execute_reply":"2024-01-23T08:06:49.190315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class FashionModel(nn.Module):\n    def __init__(self, num_classes=cfg.num_classes, pretrained=True):\n        super().__init__()\n        self.backbone = timm.create_model(\"swin_base_patch4_window12_384\", pretrained=pretrained)\n        self.in_features = self.backbone.head.fc.in_features\n        self.backbone.head.fc = nn.Sequential(nn.Dropout(cfg.cls_drop),\n                                              nn.Linear(self.in_features, num_classes))\n        \n    def forward(self, x):\n        logits = self.backbone(x)\n        return logits\n        ","metadata":{"execution":{"iopub.status.busy":"2024-01-23T10:57:56.275204Z","iopub.execute_input":"2024-01-23T10:57:56.275926Z","iopub.status.idle":"2024-01-23T10:57:56.282616Z","shell.execute_reply.started":"2024-01-23T10:57:56.275887Z","shell.execute_reply":"2024-01-23T10:57:56.281668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optimizer and Scheduler","metadata":{}},{"cell_type":"code","source":"def get_optimizer(params):\n    model_optimizer = torch.optim.AdamW(\n            filter(lambda p: p.requires_grad, params), \n            lr=cfg.lr,\n            weight_decay=cfg.wd\n        )\n    \n    lr_scheduler = CosineAnnealingWarmRestarts(\n                            model_optimizer, \n                            T_0=cfg.epochs, \n                            T_mult=1, \n                            eta_min=1e-6, \n                            last_epoch=-1\n                        )\n\n    return model_optimizer, lr_scheduler","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:07:01.468538Z","iopub.execute_input":"2024-01-23T08:07:01.469478Z","iopub.status.idle":"2024-01-23T08:07:01.475581Z","shell.execute_reply.started":"2024-01-23T08:07:01.46943Z","shell.execute_reply":"2024-01-23T08:07:01.474479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T11:08:21.484737Z","iopub.execute_input":"2024-01-23T11:08:21.485134Z","iopub.status.idle":"2024-01-23T11:08:21.644515Z","shell.execute_reply.started":"2024-01-23T11:08:21.485097Z","shell.execute_reply":"2024-01-23T11:08:21.643478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T11:09:04.565249Z","iopub.execute_input":"2024-01-23T11:09:04.566098Z","iopub.status.idle":"2024-01-23T11:09:34.981813Z","shell.execute_reply.started":"2024-01-23T11:09:04.566064Z","shell.execute_reply":"2024-01-23T11:09:34.98073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training with Stratified split","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, random_state=69, shuffle=True)\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(df_train, df_train['label'])):\n    t_df = df_train.loc[train_idx].reset_index(drop=True)\n    v_df = df_train.loc[val_idx].reset_index(drop=True)\n    \n    # datasets\n    train_dataset = FashionDataset(df=t_df)\n    valid_dataset = FashionDataset(df=v_df, train=False)\n    \n    # dataloaders\n    train_loader = DataLoader(train_dataset, batch_size=cfg.bs, num_workers=2, pin_memory=True, shuffle=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=cfg.bs*2, num_workers=2, pin_memory=True, shuffle=True)\n    \n    # model\n    model = FashionModel()\n    model = nn.DataParallel(model, device_ids=[0, 1])\n    model = model.cuda()\n    scaler = amp.GradScaler()\n    criterion = nn.CrossEntropyLoss()  # Removed label_smoothing=cfg.ls\n    \n    # optimizer & scheduler\n    optimizer, scheduler = get_optimizer(model.parameters())\n    \n    best_val_loss = np.inf\n    for epoch in range(1, cfg.epochs+1):\n        model.train()\n        cur_lr = f\"LR : {scheduler.get_last_lr()[0]:.2E}\"\n        pbar_train = enumerate(train_loader)\n        pbar_train = tqdm(pbar_train, total=len(train_loader), bar_format=\"{l_bar}{bar:10}{r_bar}{bar:-10b}\")\n        loss_train, loss_val, train_acc, val_acc = 0.0, 0.0, 0.0, 0.0\n        \n        train_preds = []\n        train_labels = []\n        for i, (imgs, labels) in pbar_train:\n            imgs, labels = imgs.cuda(), labels.cuda()\n            optimizer.zero_grad()\n            with amp.autocast():\n                preds = model(imgs)\n                loss = criterion(preds, labels)\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n                loss_train += loss.detach().item()\n            \n            train_preds.append(preds.detach().cpu().numpy())\n            train_labels.append(labels.detach().cpu().numpy())\n                \n            gpu_mem = f\"Mem : {torch.cuda.memory_reserved() / 1E9:.3g}GB\"\n            pbar_train.set_description((\"%10s  \" * 3 + \"%10s\") % (f\"Epoch {epoch}/{cfg.epochs}\", gpu_mem, cur_lr,\n                                                              f\"Loss: {loss_train / (i + 1):.4f}\"))\n        \n        train_preds = np.argmax(np.concatenate(train_preds), axis=-1)\n        train_labels = np.concatenate(train_labels)\n        train_acc = accuracy_score(train_labels, train_preds)\n        scheduler.step()\n        model.eval()\n        \n        pbar_val = enumerate(valid_loader)\n        pbar_val = tqdm(pbar_val, total=len(valid_loader), bar_format=\"{l_bar}{bar:10}{r_bar}{bar:-10b}\")\n\n        val_preds = []\n        val_labels = []\n        for i, (imgs, labels) in pbar_val:\n            imgs, labels = imgs.cuda(), labels.cuda()\n            with torch.no_grad():\n                preds = model(imgs)\n                loss_val += criterion(preds, labels).item()\n                \n            val_preds.append(preds.detach().cpu().numpy())\n            val_labels.append(labels.detach().cpu().numpy())\n            \n            pbar_val.set_description((\"%10s\") % (f\"Val Loss: {loss_val / (i+1):.4f}\"))\n            \n        val_preds = np.argmax(np.concatenate(val_preds), axis=-1)\n        val_labels = np.concatenate(val_labels)\n        val_acc = accuracy_score(val_labels, val_preds)\n        \n        print(\"#\"*10, f\"Fold/Epoch: {fold+1}/{epoch} | Train Acc: {train_acc} | Val Acc: {val_acc}\", \"#\"*10)\n        val_epoch_loss = ((loss_val)/len(valid_loader))         \n        if val_epoch_loss < best_val_loss:\n            best_val_loss = val_epoch_loss\n            torch.save(model.state_dict(), f\"best_foldswim{fold}.pt\")","metadata":{"execution":{"iopub.status.busy":"2024-01-23T11:09:40.19112Z","iopub.execute_input":"2024-01-23T11:09:40.191871Z","iopub.status.idle":"2024-01-23T11:09:44.399751Z","shell.execute_reply.started":"2024-01-23T11:09:40.191836Z","shell.execute_reply":"2024-01-23T11:09:44.397579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}