{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"> Thanks to [seefun](https://www.kaggle.com/seefun) and his [TorchUtils](https://github.com/seefun/TorchUtils)\n\nAt first, I tried only first three folds and thus create the dataset (`../input/three-pretrained-resnestpth`)\n\nLater on, I further trained more epochs for the first three folds, and considered the last two folds. (`but it was already ddl` )\n\nMy final result (with only three folds, which is obviously improvable) is 0.98272/19 for public LB and 0.98477/19 for private LB. ","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/torchutils-l/TorchUtils-master/torch_utils')\n!pip install -r ../input/torchutils-l/TorchUtils-master/requirements.txt\n!pip install ../input/torchutils-l/TorchUtils-master/\nimport os\nimport cv2\nimport time\nimport math\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd \nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom torch.optim import Adam, AdamW\nfrom torch.nn.parameter import Parameter\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader, Dataset\nfrom sklearn import metrics\nimport urllib\nimport pickle\nimport torch.nn.functional as F\nimport seaborn as sns\nimport random\nimport sys\nimport gc\nimport shutil\nfrom tqdm.autonotebook import tqdm\nimport albumentations\nfrom albumentations import pytorch as AT\n\nimport scipy.special\nsigmoid = lambda x: scipy.special.expit(x)\nfrom scipy.special import softmax\n\nimport torch_utils as tu \n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-26T17:04:58.199879Z","iopub.execute_input":"2021-06-26T17:04:58.200292Z","iopub.status.idle":"2021-06-26T17:05:24.548612Z","shell.execute_reply.started":"2021-06-26T17:04:58.200212Z","shell.execute_reply":"2021-06-26T17:05:24.547694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('../input/three-pretrained-resnestpth')","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.550149Z","iopub.execute_input":"2021-06-26T17:05:24.550461Z","iopub.status.idle":"2021-06-26T17:05:24.55571Z","shell.execute_reply.started":"2021-06-26T17:05:24.550422Z","shell.execute_reply":"2021-06-26T17:05:24.554905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\nbase_dir = '../input/d2lclassifyleaves/'\ntu.tools.seed_everything(SEED, deterministic=False)\ntu.tools.set_gpus('3') # gpu ids\nEXP = 1\nwhile os.path.exists('./exp/exp%d'%EXP):\n    EXP+=1\nos.makedirs('./exp/exp%d'%EXP)\nCLASSES = 176\nFOLD = 5\nBATCH_SIZE = 64\nACCUMULATE = 1\nLR = 3e-4\nEPOCH = 35\nDECAY_SCALE = 20.0\nMIXUP = 0.2 # 0 to 1","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.557695Z","iopub.execute_input":"2021-06-26T17:05:24.558292Z","iopub.status.idle":"2021-06-26T17:05:24.570953Z","shell.execute_reply.started":"2021-06-26T17:05:24.558257Z","shell.execute_reply":"2021-06-26T17:05:24.570149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = albumentations.Compose([\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.0625, rotate_limit=45, border_mode=1, p=0.5),\n    tu.randAugment(),\n    albumentations.Normalize(),\n    AT.ToTensorV2(),\n    ])\n    \ntest_transform = albumentations.Compose([\n    albumentations.Normalize(),\n    AT.ToTensorV2(),\n    ])\n\n\nclass LeavesDataset(Dataset):\n    \n    def __init__(self, df, label_encoder, data_path='../input/d2lclassifyleaves/images', transform = train_transform): \n        self.df = df \n        self.data_path = data_path\n        self.transform = transform\n        self.df.label = self.df.label.apply(lambda x: label_encoder[x])\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        img_path, label = self.df.image[idx], self.df.label[idx]\n        img_path = os.path.join(self.data_path, img_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.transform(image = img)['image']\n        return img, label","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.572628Z","iopub.execute_input":"2021-06-26T17:05:24.572991Z","iopub.status.idle":"2021-06-26T17:05:24.584724Z","shell.execute_reply.started":"2021-06-26T17:05:24.572943Z","shell.execute_reply":"2021-06-26T17:05:24.583906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(base_dir, 'train.csv'))\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.585882Z","iopub.execute_input":"2021-06-26T17:05:24.58634Z","iopub.status.idle":"2021-06-26T17:05:24.641887Z","shell.execute_reply.started":"2021-06-26T17:05:24.586308Z","shell.execute_reply":"2021-06-26T17:05:24.641192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sfolder = StratifiedKFold(n_splits=FOLD,random_state=SEED,shuffle=True)\ntr_folds = []\nval_folds = []\nfor train_idx, val_idx in sfolder.split(train_df.image, train_df.label):\n    tr_folds.append(train_idx)\n    val_folds.append(val_idx)\n    print(len(train_idx), len(val_idx))","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.642998Z","iopub.execute_input":"2021-06-26T17:05:24.643327Z","iopub.status.idle":"2021-06-26T17:05:24.698285Z","shell.execute_reply.started":"2021-06-26T17:05:24.643292Z","shell.execute_reply":"2021-06-26T17:05:24.697559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim.lr_scheduler import CosineAnnealingLR\nscaler = torch.cuda.amp.GradScaler() # for AMP training","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.699369Z","iopub.execute_input":"2021-06-26T17:05:24.699756Z","iopub.status.idle":"2021-06-26T17:05:24.704232Z","shell.execute_reply.started":"2021-06-26T17:05:24.699715Z","shell.execute_reply":"2021-06-26T17:05:24.70321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(epoch, verbose=False):\n    model_conv.train()         \n    avg_loss = 0.\n    optimizer.zero_grad()\n    if verbose:\n        bar = tqdm(total=len(train_loader))\n    mixup_fn = tu.Mixup(prob=MIXUP, switch_prob=0.0, onehot=True, label_smoothing=0.05, num_classes=CLASSES)\n    for idx, (imgs, labels) in enumerate(train_loader):\n        imgs_train, labels_train = imgs.float().cuda(), labels.cuda()\n        if MIXUP:\n            imgs_train, labels_train = mixup_fn(imgs_train, labels_train)\n        with torch.cuda.amp.autocast():\n            output_train, _ = model_conv(imgs_train)\n            loss = criterion(output_train, labels_train)\n        scaler.scale(loss).backward()\n        if ((idx+1)%ACCUMULATE==0): # Gradient Accumulate\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            scheduler.step()\n        avg_loss += loss.item() / len(train_loader) \n        if verbose:\n            bar.update(1)\n    if verbose:\n        bar.close()\n    return avg_loss\n\ndef test_model():    \n    avg_val_loss = 0.\n    model_conv.eval()\n    y_true_val = np.zeros(len(valset))\n    y_pred_val = np.zeros((len(valset), CLASSES))\n    with torch.no_grad():\n        for idx, (imgs, labels) in enumerate(val_loader):\n            imgs_vaild, labels_vaild = imgs.float().cuda(), labels.cuda()\n            output_test, _ = model_conv(imgs_vaild)\n            avg_val_loss += (criterion_test(output_test, labels_vaild).item() / len(val_loader)) \n            a = labels_vaild.detach().cpu().numpy().astype(np.int)\n            b = softmax(output_test.detach().cpu().numpy(), axis=1)\n\n            y_true_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = a\n            y_pred_val[idx*BATCH_SIZE:idx*BATCH_SIZE+b.shape[0]] = b\n            \n    metric_val = sum(np.argmax(y_pred_val, axis=1) == y_true_val) / len(y_true_val)\n    return avg_val_loss, metric_val","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.707113Z","iopub.execute_input":"2021-06-26T17:05:24.707471Z","iopub.status.idle":"2021-06-26T17:05:24.722004Z","shell.execute_reply.started":"2021-06-26T17:05:24.707436Z","shell.execute_reply":"2021-06-26T17:05:24.721163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(fold):\n    best_avg_loss = 100.0\n    best_acc = 0.0\n\n    avg_val_loss, avg_val_acc = test_model()\n    print('pretrain val loss %.4f precision %.4f'%(avg_val_loss, avg_val_acc))       \n\n    ### training\n    for epoch in range(EPOCH):   \n        print('lr:', optimizer.param_groups[0]['lr']) \n        np.random.seed(SEED+EPOCH*999)\n        start_time = time.time()\n        avg_loss = train_model(epoch)\n        avg_val_loss, avg_val_acc = test_model()\n        elapsed_time = time.time() - start_time \n        print('Epoch {}/{} \\t train_loss={:.4f} \\t val_loss={:.4f} \\t val_precision={:.4f} \\t time={:.2f}s'.format(\n            epoch + 1, EPOCH, avg_loss, avg_val_loss, avg_val_acc, elapsed_time))\n\n        if avg_val_loss < best_avg_loss:\n            best_avg_loss = avg_val_loss\n\n        if avg_val_acc > best_acc:\n            best_acc = avg_val_acc\n            torch.save(model_conv.module.state_dict(), './exp/exp' + str(EXP) + '/model-best' + str(fold) + '.pth')\n            print('model saved!')\n\n        print('=================================')   \n\n    print('best loss:', best_avg_loss)\n    print('best precision:', best_acc)\n    return best_avg_loss, best_acc","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.723489Z","iopub.execute_input":"2021-06-26T17:05:24.723826Z","iopub.status.idle":"2021-06-26T17:05:24.734984Z","shell.execute_reply.started":"2021-06-26T17:05:24.723794Z","shell.execute_reply":"2021-06-26T17:05:24.734123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(os.path.join(base_dir, 'test.csv'))\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.736223Z","iopub.execute_input":"2021-06-26T17:05:24.736614Z","iopub.status.idle":"2021-06-26T17:05:24.764097Z","shell.execute_reply.started":"2021-06-26T17:05:24.736579Z","shell.execute_reply":"2021-06-26T17:05:24.763346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LeavesDataset_test(Dataset):\n    \n    def __init__(self, df, data_path='../input/d2lclassifyleaves/', transform = test_transform): \n        self.df = df \n        self.data_path = data_path\n        self.transform = transform\n#         self.df.label = self.df.label.apply(lambda x: label_encoder[x])\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, idx):\n        img_path = self.df.image[idx]\n        img_path = os.path.join(self.data_path, img_path)\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.transform(image = img)['image']\n        return img #, label","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.765198Z","iopub.execute_input":"2021-06-26T17:05:24.765533Z","iopub.status.idle":"2021-06-26T17:05:24.772688Z","shell.execute_reply.started":"2021-06-26T17:05:24.765501Z","shell.execute_reply":"2021-06-26T17:05:24.771886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = LeavesDataset_test(test_df)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=False, num_workers=8)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.773769Z","iopub.execute_input":"2021-06-26T17:05:24.774132Z","iopub.status.idle":"2021-06-26T17:05:24.783887Z","shell.execute_reply.started":"2021-06-26T17:05:24.7741Z","shell.execute_reply":"2021-06-26T17:05:24.783042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################### Model ########################\nmodel_conv = tu.ImageModel(name='resnest50d', pretrained=False, feature=2048, classes=CLASSES)\nmodel_conv.eval()\nmodel_conv.load_state_dict(torch.load('../input/three-pretrained-resnestpth/model-best0.pth'))\nmodel_conv.cuda()\nmodel_conv = torch.nn.DataParallel(model_conv)\n\nmodel_conv1 = tu.ImageModel(name='resnest50d', pretrained=False, feature=2048, classes=CLASSES)\nmodel_conv1.eval()\nmodel_conv1.load_state_dict(torch.load('../input/three-pretrained-resnestpth/model-best1.pth'))\nmodel_conv1.cuda()\nmodel_conv1 = torch.nn.DataParallel(model_conv1)\n\nmodel_conv2 = tu.ImageModel(name='resnest50d', pretrained=False, feature=2048, classes=CLASSES)\nmodel_conv2.eval()\nmodel_conv2.load_state_dict(torch.load('../input/three-pretrained-resnestpth/model-best2.pth'))\nmodel_conv2.cuda()\nmodel_conv2 = torch.nn.DataParallel(model_conv2)\n\nmodel_conv3 = tu.ImageModel(name='resnest50d', pretrained=False, feature=2048, classes=CLASSES)\nmodel_conv3.eval()\nmodel_conv3.load_state_dict(torch.load('../input/three-pretrained-resnestpth/model-best3.pth'))\nmodel_conv3.cuda()\nmodel_conv3 = torch.nn.DataParallel(model_conv3)\n\nmodel_conv4 = tu.ImageModel(name='resnest50d', pretrained=False, feature=2048, classes=CLASSES)\nmodel_conv4.eval()\nmodel_conv4.load_state_dict(torch.load('../input/three-pretrained-resnestpth/model-best4.pth'))\nmodel_conv4.cuda()\nmodel_conv4 = torch.nn.DataParallel(model_conv4)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:24.785288Z","iopub.execute_input":"2021-06-26T17:05:24.785643Z","iopub.status.idle":"2021-06-26T17:05:38.082629Z","shell.execute_reply.started":"2021-06-26T17:05:24.785609Z","shell.execute_reply":"2021-06-26T17:05:38.081756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = train_df.label.unique()\nlabel_encoder = {}\nfor idx, name in enumerate(labels):\n    label_encoder.update({name:idx})","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:38.084037Z","iopub.execute_input":"2021-06-26T17:05:38.084355Z","iopub.status.idle":"2021-06-26T17:05:38.091369Z","shell.execute_reply.started":"2021-06-26T17:05:38.08432Z","shell.execute_reply":"2021-06-26T17:05:38.090398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#用数字表示对应类\nclass_to_num = label_encoder\n#将数字转换回对应的类\nnum_to_class = {v: k for k, v in class_to_num.items()}","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:38.092815Z","iopub.execute_input":"2021-06-26T17:05:38.093297Z","iopub.status.idle":"2021-06-26T17:05:38.100613Z","shell.execute_reply.started":"2021-06-26T17:05:38.09325Z","shell.execute_reply":"2021-06-26T17:05:38.09986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a list to store the predictions.\npredictions = []\n# Iterate the testing set by batches.\nfor batch in tqdm(test_loader):    \n    imgs = batch\n    with torch.no_grad():\n        logits, _ = model_conv(imgs.cuda())\n        logits1, _ = model_conv1(imgs.cuda())\n        logits2, _ = model_conv2(imgs.cuda())\n        logits3, _ = model_conv3(imgs.cuda())\n        logits4, _ = model_conv4(imgs.cuda())\n        mean_logits = (logits + logits1 + logits2 + logits3 + logits4)/5\n    \n    # Take the class with greatest logit as prediction and record it.\n    predictions.extend(mean_logits.argmax(dim=-1).cpu().numpy().tolist())\n\n    \npreds = []\nfor i in predictions:\n    preds.append(num_to_class[i])\n\ntest_data = test_df\ntest_data['label'] = pd.Series(preds)\nsubmission = pd.concat([test_data['image'], test_data['label']], axis=1)\nsaveFileName = './submission.csv'\nsubmission.to_csv(saveFileName, index=False)\nprint(\"Done!!!!!!!!!!!!!!!!!!!!!!!!!!!\")","metadata":{"execution":{"iopub.status.busy":"2021-06-26T17:05:38.101905Z","iopub.execute_input":"2021-06-26T17:05:38.102307Z","iopub.status.idle":"2021-06-26T17:07:23.483909Z","shell.execute_reply.started":"2021-06-26T17:05:38.102271Z","shell.execute_reply":"2021-06-26T17:07:23.482957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}