{"metadata":{"accelerator":"GPU","colab":{"gpuType":"T4","machine_shape":"hm","provenance":[]},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":29193,"databundleVersionId":2318453,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Leaves classification (score:0.9813)\n","metadata":{}},{"cell_type":"markdown","source":"- **model**: resnext50_32x4d(0.9813), efficientnet_b4(0.9775),densenet201\n- **data augumentation**: random horizon and vertical flip, mixup+cutmix\n- **optimizer**: AdamW\n- **lr scheduler**: cosineAnnealingWarmRestarts (cosineAnnealing)\n- **epochs**: 30\n\n\n\ntodo:\n- cross validation: doesn't improve too much and 5 times computational cost\n- data clean\n- data visualization","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.backends.cudnn as cudnn\nimport torchvision\nfrom torchvision import datasets, models, transforms\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau,CosineAnnealingWarmRestarts\nfrom sklearn.model_selection import KFold\n\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nimport time\nimport os\nimport shutil\nfrom tempfile import TemporaryDirectory\nfrom tqdm import tqdm\n\ncudnn.benchmark = True\nplt.ion()","metadata":{"ExecuteTime":{"end_time":"2024-03-18T11:52:41.577477300Z","start_time":"2024-03-18T11:52:39.484426500Z"},"id":"initial_id","execution":{"iopub.status.busy":"2024-03-21T14:26:02.469575Z","iopub.execute_input":"2024-03-21T14:26:02.469963Z","iopub.status.idle":"2024-03-21T14:26:02.480217Z","shell.execute_reply.started":"2024-03-21T14:26:02.469921Z","shell.execute_reply":"2024-03-21T14:26:02.479152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-03-21T14:26:02.481313Z","iopub.execute_input":"2024-03-21T14:26:02.481583Z","iopub.status.idle":"2024-03-21T14:26:02.496035Z","shell.execute_reply.started":"2024-03-21T14:26:02.481559Z","shell.execute_reply":"2024-03-21T14:26:02.495176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Setting Random Seed for Reproducibility","metadata":{}},{"cell_type":"code","source":"import random\n\ndef set_seed(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True\n\nset_seed(42)","metadata":{"ExecuteTime":{"end_time":"2024-03-18T11:52:41.609465900Z","start_time":"2024-03-18T11:52:41.571483200Z"},"execution":{"iopub.status.busy":"2024-03-21T14:26:02.498038Z","iopub.execute_input":"2024-03-21T14:26:02.498326Z","iopub.status.idle":"2024-03-21T14:26:02.505262Z","shell.execute_reply.started":"2024-03-21T14:26:02.498303Z","shell.execute_reply":"2024-03-21T14:26:02.504462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading CSV Data and Creating Class Labels Mapping","metadata":{}},{"cell_type":"code","source":"# csv = pd.read_csv(\"/kaggle/input/classify-leaves/train.csv\")\ncsv = pd.read_csv(\"/kaggle/input/classify-leaves/train.csv\")\n\nclasses= csv.iloc[:,1]\nclass_names =sorted(list(set(classes)))\nlabel2n={}\nn2label={}\nfor i, class_name in enumerate(class_names):\n  label2n[class_name]=i\n  n2label[i] = class_name","metadata":{"ExecuteTime":{"end_time":"2024-03-18T11:52:41.811704300Z","start_time":"2024-03-18T11:52:41.617196600Z"},"id":"P0WQH0dyMK7T","execution":{"iopub.status.busy":"2024-03-21T14:26:02.506600Z","iopub.execute_input":"2024-03-21T14:26:02.507459Z","iopub.status.idle":"2024-03-21T14:26:02.534869Z","shell.execute_reply.started":"2024-03-21T14:26:02.507425Z","shell.execute_reply":"2024-03-21T14:26:02.534096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data\n### Customize dataset","metadata":{}},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    def __init__(self, csv, img_root, transform=None, is_train=False):\n        self.csv = csv\n        self.img_root = img_root\n        self.transform = transform\n        self.is_train = is_train\n\n    def __len__(self):\n        return len(self.csv['image'])\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_root, self.csv['image'][idx])\n        image = Image.open(img_path).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n\n        if self.is_train:\n          label = label2n[self.csv['label'][idx]]\n          return image, label\n        else:\n          return image","metadata":{"ExecuteTime":{"end_time":"2024-03-18T11:52:41.851760700Z","start_time":"2024-03-18T11:52:41.813815600Z"},"id":"10e399d9f279dc05","execution":{"iopub.status.busy":"2024-03-21T14:26:02.535832Z","iopub.execute_input":"2024-03-21T14:26:02.536076Z","iopub.status.idle":"2024-03-21T14:26:02.543478Z","shell.execute_reply.started":"2024-03-21T14:26:02.536055Z","shell.execute_reply":"2024-03-21T14:26:02.542548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data augmentation\n- random horizon and vertical flip\n- random mixup and cutmix （used in Dataloader）","metadata":{}},{"cell_type":"code","source":"data_transforms = {\n    'train': transforms.Compose([\n      transforms.Resize(224),\n      transforms.RandomHorizontalFlip(p=0.5),\n      transforms.RandomVerticalFlip(p=0.5),\n      transforms.ToTensor(),\n      transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),  # typically from ImageNet\n    ]),\n    'val': transforms.Compose([\n      transforms.Resize(224),\n      # transforms.CenterCrop(224),\n      transforms.ToTensor(),\n      transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),  # typically from ImageNet\n    ]),\n}\n\n# mixup & cutmix\nfrom torchvision.transforms import v2\nfrom torch.utils.data import default_collate\n\nNUM_CLASSES=176\ncutmix = v2.CutMix(num_classes=NUM_CLASSES)\nmixup = v2.MixUp(num_classes=NUM_CLASSES)\ncutmix_or_mixup = v2.RandomChoice([cutmix, mixup])\n\ndef collate_fn(batch):\n    return cutmix_or_mixup(*default_collate(batch))","metadata":{"ExecuteTime":{"end_time":"2024-03-18T11:52:41.854973400Z","start_time":"2024-03-18T11:52:41.828925200Z"},"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-03-21T14:26:02.544827Z","iopub.execute_input":"2024-03-21T14:26:02.545169Z","iopub.status.idle":"2024-03-21T14:26:02.556459Z","shell.execute_reply.started":"2024-03-21T14:26:02.545138Z","shell.execute_reply":"2024-03-21T14:26:02.555013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### K folds\nI only use the first fold to train the model since using cross validation doesn't improve too much in this scenario.","metadata":{}},{"cell_type":"code","source":"n_folds = 6\nkf = KFold(n_splits = n_folds,shuffle=True)\n\ncsv = csv.sample(frac = 1)  # shuffle the data\nfolds = [(x,y) for (x,y) in kf.split( csv['image'])]  # k fold","metadata":{"ExecuteTime":{"end_time":"2024-03-18T11:52:42.313496500Z","start_time":"2024-03-18T11:52:41.868280400Z"},"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-03-21T14:26:02.558001Z","iopub.execute_input":"2024-03-21T14:26:02.558505Z","iopub.status.idle":"2024-03-21T14:26:02.571766Z","shell.execute_reply.started":"2024-03-21T14:26:02.558451Z","shell.execute_reply":"2024-03-21T14:26:02.570563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataloader","metadata":{}},{"cell_type":"code","source":"img_root = '/kaggle/input/classify-leaves'\nbatch_size = 16\ntrain_csv = csv.iloc[folds[0][0], :].reset_index(drop=True)\nval_csv = csv.iloc[folds[0][1], :].reset_index(drop=True)\n\ntrain_set = ImageDataset(train_csv, img_root, data_transforms['train'], is_train=True)\nval_set = ImageDataset(val_csv, img_root, data_transforms['val'], is_train=True)\n\ntrain_loader = DataLoader(train_set, batch_size, shuffle=True,collate_fn=collate_fn)\nval_loader = DataLoader(val_set, batch_size, shuffle=True)\n\ndataloaders = {\"train\": train_loader, \"val\": val_loader}\ndataset_sizes = {\"train\": train_set.__len__(), \"val\": val_set.__len__()}","metadata":{"execution":{"iopub.status.busy":"2024-03-21T14:26:02.572966Z","iopub.execute_input":"2024-03-21T14:26:02.573251Z","iopub.status.idle":"2024-03-21T14:26:02.585586Z","shell.execute_reply.started":"2024-03-21T14:26:02.573223Z","shell.execute_reply":"2024-03-21T14:26:02.584460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training and validationn","metadata":{}},{"cell_type":"code","source":"def train_model(dataloaders, dataset_sizes, k,num_epochs):\n    \n#     model = models.resnext50_32x4d(weights='IMAGENET1K_V2')\n\n#     in_ft = model.fc.in_features\n#     model.fc = nn.Linear(in_ft, 176)\n#     nn.init.xavier_uniform_(model.fc.weight)\n#     model = model.to(device)\n\n    model = models.efficientnet_b5(weights='IMAGENET1K_V1')\n    in_ft = model.classifier[1].in_features\n    model.classifier[1] = nn.Linear(in_ft, 176)\n    nn.init.xavier_uniform_(model.classifier[1].weight)\n    \n    model = model.to(device)\n\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(),lr=0.001)\n    # scheduler = ReduceLROnPlateau(optimizer,patience = 2,factor=0.9)\n    scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=12)\n\n    os.makedirs('model', exist_ok=True)\n    best_model_params_path = os.path.join('model', f'model_{k}.pt')\n    since = time.time()\n\n    torch.save(model.state_dict(), best_model_params_path)\n    best_acc = 0.0\n    for epoch in range(num_epochs):\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for i,data in enumerate(tqdm(dataloaders[phase],desc=f'{phase} Epoch {epoch}/{num_epochs - 1}')):\n                inputs = data[0].to(device)\n                labels = data[1].to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                        scheduler.step(epoch+1/dataset_sizes['train'])\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                if phase =='train':\n                    running_corrects += torch.sum(preds == torch.max(labels,1)[1])\n                else:\n                    running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                torch.save(model.state_dict(), best_model_params_path)\n        time_elapsed = time.time() - since\n        # if phase=='val':\n        #     scheduler.step()\n        #     print(optimizer.param_groups[0]['lr'])\n    print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')\n    print(f'Best val Acc: {best_acc:4f}')","metadata":{"execution":{"iopub.status.busy":"2024-03-21T14:26:02.589440Z","iopub.execute_input":"2024-03-21T14:26:02.589803Z","iopub.status.idle":"2024-03-21T14:26:02.606589Z","shell.execute_reply.started":"2024-03-21T14:26:02.589770Z","shell.execute_reply":"2024-03-21T14:26:02.605669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model(dataloaders,dataset_sizes,k='ef_b5',num_epochs=36)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test","metadata":{}},{"cell_type":"markdown","source":"### Test dataloader","metadata":{}},{"cell_type":"code","source":"# test_csv = pd.read_csv('/kaggle/input/classify-leaves/test.csv')\nimport pandas as pd\nbatch_size = 32\nimg_root = '/kaggle/input/classify-leaves'\ntest_csv = pd.read_csv('/kaggle/input/classify-leaves/test.csv')\ntest_data = ImageDataset(test_csv,img_root,data_transforms[\"val\"])\ntest_loader = DataLoader(test_data, batch_size, shuffle=False)","metadata":{"collapsed":false,"is_executing":true,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load model with best performance on validation sets and test","metadata":{}},{"cell_type":"code","source":"predictions = [[] for _ in range(5)]\n# model = models.resnext50_32x4d(weights='IMAGENET1K_V2')\n# in_ft = model.fc.in_features\n# model.fc = nn.Linear(in_ft, 176)\n\nmodel = models.efficientnet_b5(weights='IMAGENET1K_V1')\nin_ft = model.classifier[1].in_features\nmodel.classifier[1] = nn.Linear(in_ft, 176)\n\ncheckpoint = torch.load(f'/kaggle/working//model/model_ef_b5.pt')\nmodel.load_state_dict(checkpoint)\nmodel = model.to(device)\n\nwith torch.no_grad():\n    for data in test_loader:\n        images = data.to(device)\n        outputs = model(images)\n\n        _, predicted = torch.max(outputs.data, 1)\n        predicted = predicted.cpu().numpy()\n        predictions[0].extend(predicted)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Find the mode in k folds","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame(predictions,index=[f'model_{x}' for x in range(5)]).T\nresult = df.mode(axis=1).rename(columns={0: 'label'})\nresult['label'] = [n2label[x] for x in result['label']]\nresult.head()","metadata":{"is_executing":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Store the Result","metadata":{}},{"cell_type":"code","source":"test_csv[\"label\"] =result['label']\nresult_path = 'result'\n\ntry:\n    shutil.rmtree(result_path)\n    print(\"Directory and its contents deleted successfully.\")\nexcept OSError as e:\n    print(f\"Error: {result_path} : {e.strerror}\")\n    \n\nos.makedirs(result_path,exist_ok=True)\ntest_csv.to_csv(\"result/result.csv\",index=False)","metadata":{"id":"-c4PK7zAoKDQ","is_executing":true,"trusted":true},"execution_count":null,"outputs":[]}]}