{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ttach","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:12:45.920473Z","iopub.execute_input":"2022-11-22T01:12:45.920844Z","iopub.status.idle":"2022-11-22T01:12:54.492249Z","shell.execute_reply.started":"2022-11-22T01:12:45.920766Z","shell.execute_reply":"2022-11-22T01:12:54.491320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://github.com/gbaydin/hypergradient-descent.git","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:12:54.495895Z","iopub.execute_input":"2022-11-22T01:12:54.496165Z","iopub.status.idle":"2022-11-22T01:13:06.122326Z","shell.execute_reply.started":"2022-11-22T01:12:54.496138Z","shell.execute_reply":"2022-11-22T01:13:06.121463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchvision==0.13.0","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:13:06.124362Z","iopub.execute_input":"2022-11-22T01:13:06.124725Z","iopub.status.idle":"2022-11-22T01:14:52.376680Z","shell.execute_reply.started":"2022-11-22T01:13:06.124686Z","shell.execute_reply":"2022-11-22T01:14:52.375662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Imports\nimport os\nimport sys\nimport glob\nimport torch\nimport torchvision\n\nimport numpy    as np\nimport datetime as dt\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport matplotlib.pyplot   as plt\n\nfrom PIL               import Image\nfrom torch.utils.data  import Dataset\nfrom torch.autograd    import Variable\nfrom torch.optim       import lr_scheduler\n\nfrom torch.utils.data  import Dataset, DataLoader\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torchvision       import transforms, datasets, models\nfrom os                import listdir, makedirs, getcwd, remove\nfrom os.path           import isfile, join, abspath, exists, isdir, expanduser\n\n\nimport pandas as pd\n\nfrom hypergrad import SGDHD, AdamHD\n\nimport ttach as tta\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-22T01:14:52.378473Z","iopub.execute_input":"2022-11-22T01:14:52.378818Z","iopub.status.idle":"2022-11-22T01:14:53.296810Z","shell.execute_reply.started":"2022-11-22T01:14:52.378781Z","shell.execute_reply":"2022-11-22T01:14:53.295988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NAME = \"SUBMISSION\"\n\nDIM_1 = 550\nDIM_2 = 500\nDIM_TEST_1 = 550\nDIM_TEST_2 = 500\n\nBATCH_SIZE = 6\n\nNUM_EPOCHS1 = 15\n\nrandom_seed = 42\nshuffle_dataset = True\nvalidation_split = .1","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:53.300026Z","iopub.execute_input":"2022-11-22T01:14:53.300408Z","iopub.status.idle":"2022-11-22T01:14:53.306389Z","shell.execute_reply.started":"2022-11-22T01:14:53.300370Z","shell.execute_reply":"2022-11-22T01:14:53.304528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Device configuration\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:53.308583Z","iopub.execute_input":"2022-11-22T01:14:53.308943Z","iopub.status.idle":"2022-11-22T01:14:53.316418Z","shell.execute_reply.started":"2022-11-22T01:14:53.308907Z","shell.execute_reply":"2022-11-22T01:14:53.315599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"../input/ammi-2021-convnets/\"\ntrain_path = join(data_path, \"train/train\")\ntest_path = join(data_path,\"test/test\")\nextraimage_path = join(data_path, \"extraimages/extraimages\")","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:53.317761Z","iopub.execute_input":"2022-11-22T01:14:53.318098Z","iopub.status.idle":"2022-11-22T01:14:53.326272Z","shell.execute_reply.started":"2022-11-22T01:14:53.318061Z","shell.execute_reply":"2022-11-22T01:14:53.325479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transformations for both the training and testing data\nmean=[0.485, 0.456, 0.406]\nstd=[0.229, 0.224, 0.225]\n\n# Do data transforms here, Try many others\ntrain_transforms = transforms.Compose([transforms.RandomRotation(30),\n                                       transforms.Resize(DIM_1),\n                                       transforms.RandomCrop(DIM_2),\n                                       transforms.RandomHorizontalFlip(0.3),\n                                       transforms.RandomVerticalFlip(0.3),\n                                       transforms.ToTensor(),\n                                       transforms.RandomErasing(0.1),\n                                       transforms.Normalize(mean=mean, std=std)])\n\ntest_transforms = transforms.Compose([ transforms.Resize(DIM_TEST_1),\n                                      transforms.CenterCrop(DIM_TEST_1),\n                                       transforms.ToTensor(),\n                                       transforms.Normalize(mean=mean, std=std)])","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:53.329765Z","iopub.execute_input":"2022-11-22T01:14:53.330323Z","iopub.status.idle":"2022-11-22T01:14:53.337611Z","shell.execute_reply.started":"2022-11-22T01:14:53.330287Z","shell.execute_reply":"2022-11-22T01:14:53.336715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, path, dim, transform=None):\n        self.classes = os.listdir(path)\n        self.path = [f\"{path}/{className}\" for className in self.classes]\n        self.file_list = [glob.glob(f\"{x}/*\") for x in self.path]\n        self.transform = transform\n        self.dim = dim\n\n        self.targets = []\n        \n\n        files = []\n        for i, className in enumerate(self.classes):\n            for fileName in self.file_list[i]:\n                files.append([i, className, fileName])\n                self.targets.append(i)\n                \n        self.file_list = files\n        files = None\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx):\n        fileName = self.file_list[idx][2]\n        classCategory = self.file_list[idx][0]\n        im = Image.open(fileName)\n        if self.transform:\n            im = self.transform(im)\n            \n        return im.view(3, self.dim, self.dim), classCategory\n\nclass CassavaTestDataset(Dataset):\n    def __init__(self, path, dim, transform=None):\n        self.classes = os.listdir(path)\n        self.path = [f\"{path}/{className}\" for className in self.classes]\n        self.file_list = [glob.glob(f\"{x}/*\") for x in self.path]\n        self.transform = transform\n        self.indices = []\n        self.dim=dim\n\n        files = []\n        for i, className in enumerate(self.classes):\n            for fileName in self.file_list[i]:\n                files.append([i, className, fileName])\n                self.indices.append(fileName.split(\"/\")[-1])\n        self.file_list = files\n        files = None\n\n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx):\n        fileName = self.file_list[idx][2]\n        index = self.file_list[idx][2]\n        im = Image.open(fileName)\n        if self.transform:\n            im = self.transform(im)\n            \n        return im.view(3, self.dim, self.dim), index\n\ntrain_data = CassavaDataset(train_path, dim=DIM_2, transform=train_transforms)\ntest_data = CassavaTestDataset(test_path, dim=DIM_TEST_1, transform=test_transforms)","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:53.338887Z","iopub.execute_input":"2022-11-22T01:14:53.339245Z","iopub.status.idle":"2022-11-22T01:14:55.490467Z","shell.execute_reply.started":"2022-11-22T01:14:53.339207Z","shell.execute_reply":"2022-11-22T01:14:55.489632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.classes","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:55.492103Z","iopub.execute_input":"2022-11-22T01:14:55.492691Z","iopub.status.idle":"2022-11-22T01:14:55.500826Z","shell.execute_reply.started":"2022-11-22T01:14:55.492652Z","shell.execute_reply":"2022-11-22T01:14:55.500114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating data indices for training and validation splits:\ndataset_size = len(train_data)\nindices = list(range(dataset_size))\nsplit = int(np.floor(validation_split * dataset_size))\n\nif shuffle_dataset :\n    np.random.seed(random_seed)\n    np.random.shuffle(indices)\n\ntrain_indices, val_indices = indices[split:], indices[:split]","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:55.502528Z","iopub.execute_input":"2022-11-22T01:14:55.503115Z","iopub.status.idle":"2022-11-22T01:14:55.512637Z","shell.execute_reply.started":"2022-11-22T01:14:55.503077Z","shell.execute_reply":"2022-11-22T01:14:55.511733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sampler = SubsetRandomSampler(train_indices)\nvalid_sampler = SubsetRandomSampler(val_indices)","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:55.513737Z","iopub.execute_input":"2022-11-22T01:14:55.514135Z","iopub.status.idle":"2022-11-22T01:14:55.523065Z","shell.execute_reply.started":"2022-11-22T01:14:55.514098Z","shell.execute_reply":"2022-11-22T01:14:55.522263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(train_data, batch_size=BATCH_SIZE,\n                                             sampler=train_sampler)\n\nvalid_loader = torch.utils.data.DataLoader(train_data, batch_size=BATCH_SIZE,\n                                             sampler=valid_sampler)\n\ntest_loader = torch.utils.data.DataLoader(test_data, batch_size=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:55.526229Z","iopub.execute_input":"2022-11-22T01:14:55.526524Z","iopub.status.idle":"2022-11-22T01:14:55.533572Z","shell.execute_reply.started":"2022-11-22T01:14:55.526482Z","shell.execute_reply":"2022-11-22T01:14:55.532582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(model, data_loader):\n    \"\"\"Measures the accuracy of a model on a data set.\"\"\" \n    # Make sure the model is in evaluation mode.\n    model.eval()\n    # We do not need to maintain intermediate activations while testing.\n    accs = []\n    with torch.no_grad():\n        \n        # Loop over test data.\n        for features, target in data_loader:\n          \n            # Forward pass.\n            output = model(features.to(device))\n            \n            # Get the label corresponding to the highest predicted probability.\n            pred = output.argmax(dim=1, keepdim=True)\n            \n            # Count number of correct predictions.\n            correct = pred.cpu().eq(target.view_as(pred)).sum().item()\n            total = pred.shape[0]\n            accs.append(correct/total)\n\n    # Print test accuracy.\n    percent = 100. * np.mean(accs)\n    st = np.std(accs)\n    return percent, st\n\ndef train(model, criterion, data_loader, test_data_loader, optimizer, num_epochs, filename):\n    \"\"\"Simple training loop for a PyTorch model.\"\"\" \n    \n    # Make sure model is in training mode.\n    model.train()\n    \n    # Move model to the device (CPU or GPU).\n    model.to(device)\n    \n    # Exponential moving average of the loss.\n    ema_loss = None\n    \n    best_acc = 0\n\n    print('----- Training Loop -----')\n    # Loop over epochs.\n    for epoch in range(num_epochs):\n        \n        # Loop over data.\n        for batch_idx, (features, target) in enumerate(data_loader):\n\n            # Forward pass.\n            output = model(features.to(device))\n            loss = criterion(output.to(device), target.to(device))\n\n            # Backward pass.\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            # NOTE: It is important to call .item() on the loss before summing.\n            if ema_loss is None:\n                ema_loss = loss.item()\n            else:\n                ema_loss += (loss.item() - ema_loss) * 0.01 \n\n        # Print out progress the end of epoch.\n        print('----- Model Evaluation -----')\n        print('Epoch: {}/{} \\tTrain Loss: {:.6f}'.format(epoch+1,num_epochs, ema_loss))\n        train_a, train_st = test(model,data_loader)\n        test_a, test_st = test(model,test_data_loader)\n        print(f'Train accuracy: ({train_a:.2f}%) with std:({train_st:.2f})')\n        print(f'Test accuracy: ({test_a:.2f}%) with std:({test_st:.2f})')\n        if test_a > best_acc:\n            best_acc = test_a\n            torch.save(model.state_dict(), filename+\".pth\")\n            \n    checkpoint = torch.load(filename+\".pth\")\n    model.load_state_dict(checkpoint)\n    print(\"------\")\n    test_a, test_st = test(model,test_data_loader)\n    print(f'Final test accuracy: ({test_a:.2f}%) with std:({test_st:.2f})')\n    \n    return model\n    \n\ndef generate_predictions(model,data_loader):\n    model.eval()\n    preds=[]\n    idx=[]\n\n    print('----- MAKING PREDICTIONS -----')\n    # We do not need to maintain intermediate activations while testing.\n    with torch.no_grad():\n        \n        # Loop over test data.\n        for features, indices in data_loader:\n            \n            # Forward pass.\n            output = model(features.to(device))\n            \n            # Get the label corresponding to the highest predicted probability.\n            pred = output.argmax(dim=1, keepdim=True)\n            for p,ind in zip(pred,indices):\n                idx.append(ind)\n                preds.append(p.item())\n\n    return preds,idx\n\ndef map_to_classes(n):\n    return train_data.classes[n]","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:14:55.534945Z","iopub.execute_input":"2022-11-22T01:14:55.535533Z","iopub.status.idle":"2022-11-22T01:14:55.550850Z","shell.execute_reply.started":"2022-11-22T01:14:55.535497Z","shell.execute_reply":"2022-11-22T01:14:55.549934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#(classifier): Sequential(\n#    (0): LayerNorm2d((768,), eps=1e-06, elementwise_affine=True)\n#    (1): Flatten(start_dim=1, end_dim=-1)\n#    (2): Linear(in_features=768, out_features=1000, bias=True)\n#  )\n\ndef get_resnext():\n    model = torchvision.models.convnext_base(torchvision.models.ConvNeXt_Base_Weights.IMAGENET1K_V1)\n    model.classifier[2] = nn.Sequential(\n        nn.Unflatten(1, torch.Size([1, 1024])),\n        nn.Conv1d(1, 5, 1),\n        nn.MaxPool1d(1024),\n        nn.Flatten(start_dim=1, end_dim=-1)\n        #nn.Linear(768,5)\n    )\n\n    criterion = torch.nn.CrossEntropyLoss()\n    optimizer = AdamHD(model.parameters(), lr=1e-4, hypergrad_lr=1e-9)#torch.optim.Adam(model.parameters())\n\n    return model, criterion, optimizer","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:22:01.418457Z","iopub.execute_input":"2022-11-22T01:22:01.418806Z","iopub.status.idle":"2022-11-22T01:22:01.427134Z","shell.execute_reply.started":"2022-11-22T01:22:01.418775Z","shell.execute_reply":"2022-11-22T01:22:01.426278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:22:02.967435Z","iopub.execute_input":"2022-11-22T01:22:02.967761Z","iopub.status.idle":"2022-11-22T01:22:02.971999Z","shell.execute_reply.started":"2022-11-22T01:22:02.967732Z","shell.execute_reply":"2022-11-22T01:22:02.971097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1, criterion, optimizer = get_resnext()\n\nmodel1 = train(model1, criterion, train_loader, valid_loader, optimizer, num_epochs=NUM_EPOCHS1, filename=\"resnext\")\n\ntta_model = tta.ClassificationTTAWrapper(model1, tta.aliases.five_crop_transform(DIM_TEST_2,DIM_TEST_2))\npredictions, _ = generate_predictions(tta_model,test_loader)\npreds.append(predictions)","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:22:08.795010Z","iopub.execute_input":"2022-11-22T01:22:08.795370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = np.mean(preds,axis=0)\nfinal_predictions.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:17:16.000296Z","iopub.status.idle":"2022-11-22T01:17:16.000698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = pd.DataFrame({\n    \"Category\": final_predictions,\n    \"Id\": test_data.indices\n})\nss.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-22T01:17:16.001826Z","iopub.status.idle":"2022-11-22T01:17:16.002594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss[\"Category\"] = predictions\nss[\"Category\"] = ss[\"Category\"].apply(map_to_classes)\nss.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.to_csv(NAME+\".csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}