{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Summary","metadata":{}},{"cell_type":"markdown","source":"In this competition, I want to try the overall strategy used in AutoGluon: stacking and k-fold bagging, which we have learned from our first competition (CA house price prediction). I did not perform repeated k-fold bagging, since it will be too costly.\nDuring the competition, I have trained 11 different models for five folds and finally selected 7 of them to develop a mean stacking ensemble model. And this model helps me to get the 9th place (tie for the 8th place) in the private LB.\n\nThe 11 models are the following:\n\n0, resnet50d, input_size: 224 (selected)\n\n1, efficientnet_b3, input_size: 224 (selected)\n\n2, resnext50_32x4d, input_size: 224 \n\n3, inception_resnet_v2, input_size: 299 (selected)\n\n4, vit_base_patch16_224, input_size: 224 (selected)\n\n5, tf_efficientnet_b3_ns, input_size: 224\n\n6, tf_efficientnet_b4_ns, input_size: 380 (selected)\n\n7, resnest200e, input_size 320 (selected)\n\n8, mixnet_s, input_size 224\n\n9, mixnet_xl, input_size 224 (selected)\n\n10, resnest50d, input_size 224","metadata":{}},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# two additional libraries that you will need to install for this notebook if you run it on kaggle or colab\n# !pip install timm\n# !pip install madgrad","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD\nfrom torchvision import datasets, models, transforms\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport cv2\nimport timm\n\nimport time\nimport os\nimport copy\nfrom tqdm import tqdm\nimport random\nfrom madgrad import MADGRAD\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Data Loading\n# ====================================================\ntrain = pd.read_csv('../input/classify-leaves/train.csv')\n\ndef get_image_file_path(image_path):\n    INPUT_DIR = '../input/classify-leaves/'\n    return INPUT_DIR+image_path\n\n# ====================================================\n# Leave labels mapping\n# ====================================================\nspecies_name_list = sorted(set(train['label']))\nspecies_to_num = dict(zip(species_name_list, range(len(species_name_list))))\nnum_to_species = {value : key for key, value in species_to_num.items()}\nnum_class = len(species_name_list)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug=False\n    print_freq=100\n    num_workers=4\n    model_name='inception_resnet_v2'\n    size=299\n    epochs=50 # not to exceed 9h\n    batch_size=32\n    learning_rate=1e-4\n    weight_decay=1e-9\n    scheduler='ReduceLROnPlateau' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    factor=0.2 # ReduceLROnPlateau\n    patience=4 # ReduceLROnPlateau\n    eps=1e-6 # ReduceLROnPlateau\n    T_max=20 # CosineAnnealingLR\n    #T_0=4 # CosineAnnealingWarmRestarts\n    min_lr=1e-6 # ['CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    seed=42 #[42,2021]\n    n_fold=5\n    train=True","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef init_logger(log_file='./train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# CV split\n# ====================================================\nfolds = train.copy()\nFold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds['label'])):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\nprint(folds.groupby(['fold']).size())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\n# note: here I made a small mistake. I did not notice the cv2.imread generate BGR image until the last week when I went through other's codes, so most of my models were trained using BGR image\n# I think using RGB image may make the training converge faster when we start from imagenet pre-trained model \nclass TrainDataset(Dataset):\n    def __init__(self, df, species_to_num, transform=None):\n        super().__init__()\n        self.df = df\n        self.species_to_num = species_to_num\n        self.file_paths = get_image_file_path(df['image'].values)\n        self.labels = df['label'].values\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        file_path = self.file_paths[idx]\n        image = cv2.imread(file_path)\n#         image = Image.open(file_path)\n        if self.transform:\n            image = self.transform(image)\n        label = self.labels[idx]\n        label = self.species_to_num[label]\n        return image, label\n    \n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        super().__init__()\n        self.df = df\n        self.file_paths = get_image_file_path(df['image'].values)\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        file_path = self.file_paths[idx]\n        image = cv2.imread(file_path)\n#         image = Image.open(file_path)\n        if self.transform:\n            image = self.transform(image)\n        return image","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Data transforms\n# ====================================================\n# I applied slightly different transforms for different groups of models\n# resnet50d, efficientnet_b3: flip only\n# resnext50_32x4d, resnest50d tf_efficientnet_b4_ns, resnest200e, mixnet_s: add ColorJitter\n# inception_resnet_v2, vit_base_patch16_224, tf_efficientnet_b3_ns: use RandomResizedCrop instead of Resize, and use [0.5, 0.5, 0.5] as mean and std\ndata_transforms = {\n    'train': transforms.Compose([\n        transforms.ToTensor(),\n        transforms.RandomHorizontalFlip(p=0.5),\n        transforms.RandomVerticalFlip(p=0.5),\n        transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0),\n        transforms.RandomResizedCrop([CFG.size, CFG.size]),\n#         transforms.Resize([CFG.size, CFG.size]),\n        transforms.Normalize(\n            mean=[0.5, 0.5, 0.5],\n            std=[0.5, 0.5, 0.5],\n#             mean=[0.485, 0.456, 0.406],\n#             std=[0.229, 0.224, 0.225],\n        ),\n        #transforms.RandomErasing(),\n    ]),\n    'valid': transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Resize([CFG.size, CFG.size]),\n        transforms.Normalize(\n            mean=[0.5, 0.5, 0.5],\n            std=[0.5, 0.5, 0.5],\n#             mean=[0.485, 0.456, 0.406],\n#             std=[0.229, 0.224, 0.225],\n        ),\n    ]),\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# scheduler \n# ====================================================\ndef get_scheduler(optimizer):\n    if CFG.scheduler=='ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n    elif CFG.scheduler=='CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n    elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n    return scheduler","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if run this on kaggle, remember to create a folder to save models\n# !mkdir models","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Model\n# ====================================================\n# def leave_classifier(model_name, num_classes, fixed_feature_extractor = True, pretrained = True):\n#     model = timm.create_model(model_name, pretrained=pretrained)\n#     if fixed_feature_extractor:\n#         for param in model.parameters():\n#             param.requires_grad = False\n#     # for resnet        \n# #     num_ftrs = model.fc.in_features\n# #     model.fc = nn.Linear(num_ftrs, num_classes)\n#     # for efficientnet\n#     num_ftrs = model.classifier.in_features\n#     model.classifier = nn.Linear(num_ftrs, num_classes)\n#     return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# training \n# ====================================================\ndef train_model(fold):\n    since = time.time()\n    model_path = './models/' + CFG.model_name + '_fold' + str(fold) + '_best.pth'\n    LOGGER.info(f\"============================== fold: {fold} result ==============================\")\n    # ====================================================\n    # Data Loader\n    # ====================================================\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n    valid_labels = valid_folds['label'].values\n    train_dataset = TrainDataset(train_folds, species_to_num, transform=data_transforms['train'])\n    valid_dataset = TrainDataset(valid_folds, species_to_num, transform=data_transforms['valid'])\n\n    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, \n                              shuffle=True, num_workers=CFG.num_workers, \n                              pin_memory=True, drop_last=True,)\n    valid_loader = DataLoader(valid_dataset, batch_size=CFG.batch_size, \n                              shuffle=False, num_workers=CFG.num_workers,\n                              pin_memory=True, drop_last=False)\n    \n    # ====================================================\n    # model, optimizer, scheduler & loss function\n    # ====================================================   \n    model = timm.create_model(CFG.model_name, pretrained=True, num_classes=num_class)\n    # use the following model function if you want to fine-tune the last layer only, which is not what I did here. But I explored it during the competition.\n    #model = leave_classifier(CFG.model_name, num_class)\n    model = model.to(device)\n    \n    #optimizer = Adam(model.parameters(), lr=CFG.learning_rate, weight_decay=CFG.weight_decay, amsgrad=False)\n    optimizer = MADGRAD(model.parameters(), lr=CFG.learning_rate, weight_decay=CFG.weight_decay)\n    scheduler = get_scheduler(optimizer)\n    \n    criterion = nn.CrossEntropyLoss()\n    # ====================================================\n    # loop\n    # ====================================================  \n    \n    best_acc = 0.95 # do not save the model if the acc is less than 0.95\n\n    for epoch in range(CFG.epochs):\n        \n        # ---------- Training ----------\n        # Make sure the model is in train mode before training.\n        model.train()\n        # These are used to record information in training.\n        train_loss = []\n        train_accs = []\n\n        global_step = 0\n        # Iterate the training set by batches.\n        for step, (imgs, labels) in enumerate(train_loader):\n            # A batch consists of image data and corresponding labels.\n            imgs = imgs.to(device)\n            labels = labels.to(device)\n            # Forward the data. (Make sure data and model are on the same device.)\n            logits = model(imgs)\n            # Calculate the cross-entropy loss.\n            # We don't need to apply softmax before computing cross-entropy as it is done automatically.\n            loss = criterion(logits, labels)\n\n            # Gradients stored in the parameters in the previous step should be cleared out first.\n            optimizer.zero_grad()\n            # Compute the gradients for parameters.\n            loss.backward()\n            # Update the parameters with computed gradients.\n            optimizer.step()\n\n            # Compute the accuracy for current batch.\n            acc = (logits.argmax(dim=-1) == labels).float().mean()\n\n            # Record the loss and accuracy.\n            train_loss.append(loss.item())\n            train_accs.append(acc)\n\n        # The average loss and accuracy of the training set is the average of the recorded values.\n        train_loss = sum(train_loss) / len(train_loss)\n        train_acc = sum(train_accs) / len(train_accs)\n\n\n        # ---------- Validation ----------\n        # Make sure the model is in eval mode so that some modules like dropout are disabled and work normally.\n        model.eval()\n        # These are used to record information in validation.\n        valid_loss = []\n        valid_accs = []\n\n        # Iterate the validation set by batches.\n        for step, (imgs, labels) in enumerate(valid_loader):\n            # We don't need gradient in validation.\n            # Using torch.no_grad() accelerates the forward process.\n            with torch.no_grad():\n                logits = model(imgs.to(device))\n\n            # We can still compute the loss (but not the gradient).\n            loss = criterion(logits, labels.to(device))\n\n            # Compute the accuracy for current batch.\n            acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()\n\n            # Record the loss and accuracy.\n            valid_loss.append(loss.item())\n            valid_accs.append(acc)\n\n        # The average loss and accuracy for entire validation set is the average of the recorded values.\n        valid_loss = sum(valid_loss) / len(valid_loss)\n        valid_acc = sum(valid_accs) / len(valid_accs)\n        \n        \n        # learning rate update\n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(valid_acc)\n        else:\n            scheduler.step()\n        \n        elapsed = time.time() - since\n        \n        # Print the information.\n        LOGGER.info(f'Epoch {epoch+1}/{CFG.epochs}: train_loss = {train_loss:.4f}, valid_loss = {valid_loss:.4f}, train_acc = {train_acc:.4f}, valid_acc = {valid_acc:.4f}, time: {elapsed:.0f}s')\n        #print(f'learning_rate = {scheduler.optimizer.param_groups[0]['lr']}')\n        # if the model improves, save a checkpoint at this epoch\n        \n        if valid_acc > best_acc:\n            best_acc = valid_acc\n            torch.save(model.state_dict(), model_path)\n            LOGGER.info(f'Save Best Score: {best_acc:.4f} Model to {model_path}')\n            #print('saving model with acc {:.3f}'.format(best_acc))\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# once you fix the random seeds, you can train different folds and different models on different machines, e.g., kaggle, colab, etc.\n# set appropriate epochs and the number of folds to remote training.\n# the models I used in this competition usually takes 1-4 min for each epoch on V100, and the time cost may double up on other GPUs like P100 and T4\n# train_model(2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model selection","metadata":{}},{"cell_type":"markdown","source":"Here I used out-of-fold predictions to develop ensemble models","metadata":{}},{"cell_type":"code","source":"def cv_prob(fold):\n    test_dataset = TestDataset(folds[folds['fold']==fold], transform=data_transforms['valid'])\n    test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    model_path = CFG.model_dir + CFG.model_name + '_fold' + str(fold) + '_best.pth'\n    model = timm.create_model(CFG.model_name, pretrained=False, num_classes=num_class)\n    model = model.to(device)\n    model.load_state_dict(torch.load(model_path))\n    model.eval()\n    \n    prob_list = []\n\n    # Iterate the testing set by batches.\n    for batch in tqdm(test_loader):\n        imgs = batch\n        with torch.no_grad():\n            logits = model(imgs.to(device))\n            prob_list.append(logits.softmax(1))\n    probs_np = torch.cat(prob_list, axis=0).to('cpu').numpy()\n    return probs_np","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(CFG.n_fold):\n    probs_np = cv_prob(fold)\n    folds.loc[(folds['fold']==fold),CFG.model_name] = np.argmax(probs_np,axis=1)\n    if(fold==0):\n        probs_np_copy = probs_np\n    else:\n        probs_np_copy = np.concatenate((probs_np_copy, probs_np), axis=0)\n    print(probs_np_copy.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# after I collected all fine-tuned models, I created a 3D array probs_3D = np.zeros([train.shape[0],num_class,num_models]) to save all softmax outputs for all the models\n# probs_3D = np.zeros([train.shape[0],num_class,num_models])\n# probs_3D[:,:,2] = probs_np_copy","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert the species name to the index for easier comparison\nfolds['label_num'] = folds['label'].map(species_to_num)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Stacked mean combinations","metadata":{}},{"cell_type":"code","source":"from itertools import combinations, chain\ncombined = []\nnum_model = probs_3D.shape[2]\nfor i in range(num_model):\n    combined.append(list(combinations(range(num_model), i+1)))\n# sort the folds to match the index used in probs_3D, the softmax output\nfold_sorted = folds.rename_axis('MyIdx').sort_values(by = ['fold', 'MyIdx'], ascending = [True, True])\n\ncomb_results = dict()\nwith tqdm(total=len(list(chain(*combined)))) as process_bar:\n    for c in list(chain(*combined)):\n        # the result indicates how many out-of-fold predictions are incorrect\n        comb_results[c] = (fold_sorted['label_num']!=np.argmax(probs_3D[:,:,c].sum(2),axis=1)).sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"{k: comb_results[k] for k in sorted(comb_results, key=comb_results.get, reverse=False)[0:20]}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Weighted average","metadata":{}},{"cell_type":"code","source":"num_model = probs_3D.shape[2]\nweights = np.array([1.0/n_models for _ in range(num_model)])\nbounds = [(0.0, 1.0) for _ in range(num_model)]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_func(w):\n    # use 1 - accuracy as the loss function to find weights\n    w= np.ceil(np.array(w)*20) # this operation is to lower the resolution of the weights \n    return (fold_sorted['label_num']!=np.argmax(np.matmul(probs_3D,w).reshape(-1, num_class),axis=1)).sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.optimize import differential_evolution\nsol = differential_evolution(loss_func, bounds, maxiter=20, tol=1e-4, disp=True)\n# sol.x is the final weight vector\n# In fact, I did not use weighted average for my submission. I used it to check the importance of each model and confirmed that stacked mean method is good enough","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('../input/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(test, transform=data_transforms['valid'])\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True, drop_last=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_prob():\n    model = timm.create_model(CFG.model_name, pretrained=False, num_classes=num_class)\n    model = model.to(device)\n    avg_preds = []\n    for fold in range(CFG.n_fold):\n        model_path = CFG.model_dir + CFG.model_name + '_fold' + str(fold) + '_best.pth'\n        model.load_state_dict(torch.load(model_path))\n        model.eval()\n        prob_list = []\n        for batch in tqdm(test_loader):\n            imgs = batch\n            with torch.no_grad():\n                logits = model(imgs.to(device))\n            prob_list.append(logits.softmax(1)) #(batch_size x num_class)\n        probs_np = torch.cat(prob_list, axis=0).to('cpu').numpy()\n        avg_preds.append(probs_np) #(test_size x num_class)\n    probs = np.mean(avg_preds, axis=0) # average of 5-fold prediction\n    return probs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I implement the method with TTA, but I did not submit any predictions with TTA\n# def test_prob_with_TTA():\n#     model = timm.create_model(CFG.model_name, pretrained=False, num_classes=num_class)\n#     model = model.to(device)\n#     avg_preds = []\n#     for fold in range(CFG.n_fold):\n#         model_path = CFG.model_dir + CFG.model_name + '_fold' + str(fold) + '_best.pth'\n#         model.load_state_dict(torch.load(model_path))\n#         model.eval()\n#         prob_list = []\n#         for batch in tqdm(test_loader):\n#             x = batch.to(device)\n#             with torch.no_grad():\n#                 x = torch.stack([x, x.flip(-1), x.flip(-2), x.flip(-1,-2),\n#                                  x.transpose(-1,-2), x.transpose(-1,-2).flip(-1),\n#                                  x.transpose(-1,-2).flip(-2), x.transpose(-1,-2).flip(-1,-2)],0)\n#                 x = x.view(-1, 3, CFG.size, CFG.size)\n#                 logits = model(x)\n#                 logits = logits.view(8, CFG.batch_size, -1).mean(0)\n#             prob_list.append(logits.softmax(1)) #(batch_size x num_class)\n#         probs_np = torch.cat(prob_list, axis=0).to('cpu').numpy()\n#         avg_preds.append(probs_np) #(test_size x num_class)\n#     probs = np.mean(avg_preds, axis=0) # average of 5-fold prediction\n#     return probs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just like what I did in model selection section, I create a 3D array to save all softmax outputs\nprobs_3D_pred = np.zeros([8800,176,11])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs_np_copy2 = test_prob()\nprobs_np_copy2.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# once the softmax outputs are saved, I do not have to run the prediction again for this model, although it only takes a few minites.\nprobs_3D_pred[:,:,0] = probs_np_copy2\n# you only need to do ensemble after doing this for all 11 models\nnp.save('prediction_raw_data_11models.npy',probs_3D_pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Uploading the models is too time-consuming. Instead, I uploaded the softmax output.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nprobs_3D_ori_11 = np.load('../input/model-predictions/prediction_raw_data_11models.npy')\ntest = pd.read_csv('../input/classify-leaves/test.csv')\ntrain = pd.read_csv('../input/classify-leaves/train.csv')\n\n# ====================================================\n# Leave labels mapping\n# ====================================================\nspecies_name_list = sorted(set(train['label']))\nspecies_to_num = dict(zip(species_name_list, range(len(species_name_list))))\nnum_to_species = {value : key for key, value in species_to_num.items()}\nnum_class = len(species_name_list)","metadata":{"execution":{"iopub.status.busy":"2021-06-27T01:21:22.791011Z","iopub.execute_input":"2021-06-27T01:21:22.791588Z","iopub.status.idle":"2021-06-27T01:21:22.928913Z","shell.execute_reply.started":"2021-06-27T01:21:22.791467Z","shell.execute_reply":"2021-06-27T01:21:22.927518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"the final submission I used is my 6th version of prediction, the mean stacking of 7 models","metadata":{}},{"cell_type":"code","source":"s6_pred = pd.Series(np.argmax(probs_3D_ori_11[:,:,(0, 1, 3, 4, 6, 7, 9)].sum(2),axis=1))","metadata":{"execution":{"iopub.status.busy":"2021-06-27T01:21:31.095027Z","iopub.execute_input":"2021-06-27T01:21:31.095448Z","iopub.status.idle":"2021-06-27T01:21:31.219637Z","shell.execute_reply.started":"2021-06-27T01:21:31.095414Z","shell.execute_reply":"2021-06-27T01:21:31.218408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['label'] = s6_pred.map(num_to_species)\nsubmission = pd.concat([test['image'], test['label']], axis=1)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-27T01:21:33.491698Z","iopub.execute_input":"2021-06-27T01:21:33.492074Z","iopub.status.idle":"2021-06-27T01:21:33.525054Z","shell.execute_reply.started":"2021-06-27T01:21:33.492042Z","shell.execute_reply":"2021-06-27T01:21:33.523883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('./submission_6.csv', index=False)\n# !kaggle competitions submit -c classify-leaves -f submission_6.csv -m \"0, 1, 3, 4, 6, 7, 9 CV98.33\"","metadata":{"execution":{"iopub.status.busy":"2021-06-27T01:21:49.296149Z","iopub.execute_input":"2021-06-27T01:21:49.297103Z","iopub.status.idle":"2021-06-27T01:21:49.341514Z","shell.execute_reply.started":"2021-06-27T01:21:49.297032Z","shell.execute_reply":"2021-06-27T01:21:49.340276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}