{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\"\"\"\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\"\"\"\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-25T14:00:43.751331Z","iopub.execute_input":"2021-06-25T14:00:43.751882Z","iopub.status.idle":"2021-06-25T14:00:43.765575Z","shell.execute_reply.started":"2021-06-25T14:00:43.751774Z","shell.execute_reply":"2021-06-25T14:00:43.764509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\n         \nzipfile.ZipFile('../input/dogs-vs-cats-redux-kernels-edition/test.zip').extractall()\nzipfile.ZipFile('../input/dogs-vs-cats-redux-kernels-edition/train.zip').extractall()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:00:43.767620Z","iopub.execute_input":"2021-06-25T14:00:43.768286Z","iopub.status.idle":"2021-06-25T14:01:03.620820Z","shell.execute_reply.started":"2021-06-25T14:00:43.768229Z","shell.execute_reply":"2021-06-25T14:01:03.619853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils import data\nfrom torchvision import datasets, transforms\nimport torch\nimport os\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.class_weight import compute_class_weight\nimport numpy as np\nimport os\nimport math\nimport datetime\n\nimport numpy as np\n\nimport time\nimport torch.nn as nn\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import StepLR\nimport torch.nn.functional as F\n\n!pip install timm\nimport timm","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:03.622606Z","iopub.execute_input":"2021-06-25T14:01:03.622918Z","iopub.status.idle":"2021-06-25T14:01:16.080050Z","shell.execute_reply.started":"2021-06-25T14:01:03.622890Z","shell.execute_reply":"2021-06-25T14:01:16.079062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 1\nLEARNING_RATE = 0.001\nBATCH_SIZE = 64\nVALID_SPLIT = 0.1\nlr_step_size = 5\nprint_iter = 10","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:48:34.204588Z","iopub.execute_input":"2021-06-25T14:48:34.204927Z","iopub.status.idle":"2021-06-25T14:48:34.211212Z","shell.execute_reply.started":"2021-06-25T14:48:34.204897Z","shell.execute_reply":"2021-06-25T14:48:34.210426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pil_loader(path, img_size=224):\n    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)\n    try:\n        with open(path, \"rb\") as f:\n            img = Image.open(f)\n            return img.convert(\"RGB\")\n    except FileNotFoundError as e:\n        raise FileNotFoundError(e)\n\n\ndef get_transform(random_crop=True):\n    normalize = transforms.Normalize(\n        mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],\n        std=[x / 255.0 for x in [63.0, 62.1, 66.7]])\n    transform = []\n    transform.append(transforms.Resize(256))\n    if random_crop:\n        transform.append(transforms.RandomHorizontalFlip())\n        transform.append(transforms.ColorJitter(brightness=0.1, saturation=0.1))\n        # transform.append(transforms.RandomPerspective())\n        transform.append(transforms.RandomResizedCrop(224))\n    else:\n        transform.append(transforms.CenterCrop(224))\n    transform.append(transforms.ToTensor())\n    transform.append(normalize)\n    return transforms.Compose(transform)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.089135Z","iopub.execute_input":"2021-06-25T14:01:16.089824Z","iopub.status.idle":"2021-06-25T14:01:16.100460Z","shell.execute_reply.started":"2021-06-25T14:01:16.089754Z","shell.execute_reply":"2021-06-25T14:01:16.099662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\n\ndef cutmix(input, target):\n    input = input.clone().detach()\n    target = target.clone().detach()\n    beta = 1.0\n    lam = np.random.beta(beta, beta)\n    rand_index = torch.randperm(input.size()[0]).cuda()\n    target_b = target[rand_index]\n    bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam)\n    input[:, :, bbx1:bbx2, bby1:bby2] = input[rand_index, :, bbx1:bbx2, bby1:bby2]\n\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n    return target_b, lam","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.102020Z","iopub.execute_input":"2021-06-25T14:01:16.102464Z","iopub.status.idle":"2021-06-25T14:01:16.114375Z","shell.execute_reply.started":"2021-06-25T14:01:16.102423Z","shell.execute_reply":"2021-06-25T14:01:16.113589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(data.Dataset):\n    def __init__(self, root='./test'):\n\n        self.root = root\n        self.samples = [[file_name, file_name.split('.')[0]] for file_name in os.listdir(self.root)]\n        self.transform = get_transform(random_crop=False)\n\n    def __getitem__(self, index):\n        '''\n        Here, our problem supposes maximum 3 hierarchy\n        '''\n        path, idx = self.samples[index]\n        path = os.path.join(self.root, path)\n        sample = self.transform(pil_loader(path=path))\n\n        return torch.LongTensor([int(idx)]), sample\n\n    def __len__(self):\n        return len(self.samples)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.115726Z","iopub.execute_input":"2021-06-25T14:01:16.116205Z","iopub.status.idle":"2021-06-25T14:01:16.125028Z","shell.execute_reply.started":"2021-06-25T14:01:16.116171Z","shell.execute_reply":"2021-06-25T14:01:16.124160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(data.Dataset):\n    def __init__(self, is_train=True, root='./train', split=1.0):\n\n        self.root = root\n        self.is_train = is_train\n                        \n        file_names = os.listdir(self.root)\n                        \n        test_size = 1.0 - split\n        if test_size > 0:\n            train_files, valid_files = train_test_split(file_names, test_size= split, random_state=97)\n        else:\n            train_files = file_names\n            valid_files = []\n\n        if is_train:\n            random_crop = True\n            dataset_files = train_files\n        else:\n            random_crop = False\n            dataset_files = valid_files\n\n        self.samples = np.array([[file_name, int(file_name.split('.')[0]=='dog'), int(file_name.split('.')[1])]\n                        for file_name in dataset_files])\n        target_names = np.unique(self.samples[:, 1])\n        self.class_weights = torch.Tensor(compute_class_weight(class_weight='balanced', \n                                                                   classes=target_names,\n                                                                   y=self.samples[:, 1]))\n        self.transform = get_transform(random_crop=random_crop)\n\n    def __getitem__(self, index):\n        path, target, idx = self.samples[index]\n        path = os.path.join(self.root, path)\n        sample = self.transform(pil_loader(path=path))\n        \n        idx, target = int(idx), int(target)\n\n        if self.is_train:\n            return torch.tensor([idx]), sample, torch.tensor(target), self.class_weights\n        return torch.tensor([idx]), sample, torch.tensor(target)\n\n    def __len__(self):\n        return len(self.samples)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:48:24.363358Z","iopub.execute_input":"2021-06-25T14:48:24.363782Z","iopub.status.idle":"2021-06-25T14:48:24.374052Z","shell.execute_reply.started":"2021-06-25T14:48:24.363746Z","shell.execute_reply":"2021-06-25T14:48:24.373207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef data_loader(is_train=True, batch_size=16, split=1.0, Test=True):\n    if Test:\n        dataset = TestDataset()\n    else:\n        dataset = CustomDataset(is_train=is_train, split=split)\n    return data.DataLoader(dataset=dataset,\n                           batch_size=batch_size,\n                           shuffle=is_train)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.143563Z","iopub.execute_input":"2021-06-25T14:01:16.143962Z","iopub.status.idle":"2021-06-25T14:01:16.152911Z","shell.execute_reply.started":"2021-06-25T14:01:16.143936Z","shell.execute_reply":"2021-06-25T14:01:16.151987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DnnClassifier(nn.Module):\n    def __init__(self, in_features=1536):\n        super(DnnClassifier, self).__init__()\n        self.fc1 = nn.Linear(in_features=in_features, out_features=512)\n        self.bn1 = nn.BatchNorm1d(512)\n        \n        self.fc2 = nn.Linear(in_features=512, out_features=128)\n        self.bn2 = nn.BatchNorm1d(128)\n\n        self.fc3 = nn.Linear(in_features=128, out_features=2)\n\n    def forward(self, x):\n        x = F.dropout(F.silu(self.bn1(self.fc1(x))))\n        x = F.dropout(F.silu(self.bn2(self.fc2(x))))\n        x = self.fc3(x)\n        x = torch.sigmoid(x)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.154408Z","iopub.execute_input":"2021-06-25T14:01:16.154802Z","iopub.status.idle":"2021-06-25T14:01:16.165955Z","shell.execute_reply.started":"2021-06-25T14:01:16.154768Z","shell.execute_reply":"2021-06-25T14:01:16.165230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MySuperUltraUniverseFCN(nn.Module):\n    def __init__(self):\n        super(MySuperUltraUniverseFCN, self).__init__()\n        self.out_features = 16\n        self.conv1 = nn.Conv2d(3, self.out_features, kernel_size=224)  # 4 * 16\n        self.bn1 = nn.BatchNorm2d(self.out_features)\n\n    def forward(self, out):\n        out = F.silu(self.bn1(self.conv1(out)))\n\n        return out","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.167320Z","iopub.execute_input":"2021-06-25T14:01:16.167707Z","iopub.status.idle":"2021-06-25T14:01:16.175824Z","shell.execute_reply.started":"2021-06-25T14:01:16.167671Z","shell.execute_reply":"2021-06-25T14:01:16.175003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsembleModel(nn.Module):\n    def __init__(self):\n        super(EnsembleModel, self).__init__()\n        self.out_features = 0\n        self.modelA = timm.create_model('tf_efficientnetv2_b3', pretrained=True)\n        self.out_features += self.modelA.classifier.in_features\n        self.modelB = timm.create_model('rexnet_100', pretrained=True)\n        self.out_features += self.modelB.head.fc.in_features\n        self.modelC = timm.create_model('tf_mobilenetv3_small_minimal_100', pretrained=True)\n        self.out_features += self.modelC.classifier.in_features\n        self.modelA.classifier = nn.Identity()\n        self.modelB.head.fc = nn.Identity()\n        self.modelC.classifier = nn.Identity()\n        self.modelFCN = MySuperUltraUniverseFCN()\n        \n        self.out_features += self.modelFCN.out_features\n        #self.classifier = DnnClassifier(in_features=self.out_features + self.modelFCN.out_features)\n        \n    def forward(self, out):\n        out1 = self.modelA(out.clone())\n        out1 = out1.view(out1.size(0), -1)\n\n        out2 = self.modelB(out.clone())\n        out2 = out2.view(out2.size(0), -1)\n\n        out3 = self.modelC(out.clone())\n        out3 = out3.view(out3.size(0), -1)\n\n        out4 = self.modelFCN(out)\n        out4 = out4.view(out4.size(0), -1)\n\n        out = torch.cat((out1, out2, out3, out4), dim=1)\n        #out = self.classifier(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.177091Z","iopub.execute_input":"2021-06-25T14:01:16.177504Z","iopub.status.idle":"2021-06-25T14:01:16.188135Z","shell.execute_reply.started":"2021-06-25T14:01:16.177472Z","shell.execute_reply":"2021-06-25T14:01:16.187236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reputation(pred, target):\n    cnt = 0\n    for pred_, target_ in zip(pred, target):\n        if torch.argmax(pred_) == target_:\n            cnt += 1\n            \n    return cnt","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.189230Z","iopub.execute_input":"2021-06-25T14:01:16.189728Z","iopub.status.idle":"2021-06-25T14:01:16.199264Z","shell.execute_reply.started":"2021-06-25T14:01:16.189693Z","shell.execute_reply":"2021-06-25T14:01:16.198233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EnsembleModel()\nfor param in model.parameters():\n    param.requires_grad = False\n\nclassifier = DnnClassifier(in_features=model.out_features)\n\ncuda = torch.cuda.is_available()\nprint('can use cuda', cuda)\n\nif cuda:\n    model = model.cuda()\n    classifier = classifier.cuda()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:01:16.200450Z","iopub.execute_input":"2021-06-25T14:01:16.200968Z","iopub.status.idle":"2021-06-25T14:01:24.319242Z","shell.execute_reply.started":"2021-06-25T14:01:16.200932Z","shell.execute_reply":"2021-06-25T14:01:24.318404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.SGD([param for param in classifier.parameters() if param.requires_grad], lr=LEARNING_RATE, momentum=0.9, weight_decay=1e-4)\nscheduler = StepLR(optimizer, step_size=lr_step_size, gamma=0.1)    ","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:48:43.259159Z","iopub.execute_input":"2021-06-25T14:48:43.259495Z","iopub.status.idle":"2021-06-25T14:48:43.264209Z","shell.execute_reply.started":"2021-06-25T14:48:43.259461Z","shell.execute_reply":"2021-06-25T14:48:43.263267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = data_loader(is_train=True, split=VALID_SPLIT, batch_size=BATCH_SIZE, Test=False)\nvalid_loader = data_loader(is_train=False, split=VALID_SPLIT, batch_size=BATCH_SIZE, Test=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:48:43.453403Z","iopub.execute_input":"2021-06-25T14:48:43.453719Z","iopub.status.idle":"2021-06-25T14:48:43.756917Z","shell.execute_reply.started":"2021-06-25T14:48:43.453692Z","shell.execute_reply":"2021-06-25T14:48:43.756076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"global_iter = 0\nfor epoch in range(EPOCHS):\n    \n    # training\n    model.train()\n    train_loss = torch.tensor(0.)\n    class_weights = [0.5, 0.5]\n    num_trained_data = 0\n    for iter_, data_ in enumerate(train_loader):\n        global_iter += iter_\n        \n        _, x, label, class_weights = data_\n        \n        num_trained_data += label.shape[0]\n        loss_fn = nn.CrossEntropyLoss(weight=class_weights[0])\n        if cuda:\n            x, label = x.cuda(), label.cuda()\n            loss_fn = loss_fn.cuda()\n            \n        pred = model(x)\n        pred = classifier(pred)\n        \n    \n        if epoch >= 10:\n            label_b, lam = cutmix(x, label)\n            train_loss = loss_fn(pred, label) * lam + loss_fn(pred, label_b) * (1. - lam)\n        else:\n            train_loss = loss_fn(pred, label)\n            \n        optimizer.zero_grad()\n        train_loss.backward()\n        optimizer.step()\n        \n        if (iter_ + 1) % print_iter == 0:\n            print('[',epoch+1,'/',EPOCHS,'] [',num_trained_data,'/',len(train_loader.dataset),'] train loss: ',float(train_loss))\n        \n    scheduler.step()\n    \n    Accuracy = 0.\n    valid_size = len(valid_loader.dataset)\n    valid_loss = torch.tensor(0.)\n    if valid_size:\n        model.eval()\n        \n        corrects = 0\n        total_score = 0\n        \n        \n        with torch.no_grad():\n            for data_ in valid_loader:\n                _, x, label, = data_\n                loss_fn = nn.CrossEntropyLoss(weight=class_weights[0])\n                \n                if cuda:\n                    x, label = x.cuda(), label.cuda()\n                    loss_fn = loss_fn.cuda()\n                    valid_loss = valid_loss.cuda()\n                    \n                pred = model(x)\n                pred = classifier(pred)\n                \n                valid_loss = loss_fn(pred, label)\n                \n                corrects += reputation(pred, label)\n        \n        Accuracy = 100 * corrects / valid_size\n    \n    print('valid loss:', float(valid_loss))\n    print('valid accuracy:', Accuracy)\n                    ","metadata":{"execution":{"iopub.status.busy":"2021-06-25T14:48:43.758926Z","iopub.execute_input":"2021-06-25T14:48:43.759792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader = data_loader(is_train=False, split=VALID_SPLIT, batch_size=BATCH_SIZE, Test=True)\n\nidx = np.array([])\npreds = np.array([])\nfor data_ in test_loader:\n    i, X_test = data_\n    if cuda:\n        x, label = x.cuda(), label.cuda()\n                    \n    pred = model(X_test)\n    pred = classifier(pred)\n    \n    preds = np.append(preds, torch.argmax(pred, dim=1).cpu().numpy())\n    idx = np.append(idx, i.numpy())\n\nd = {'id': idx, 'label':preds}\ndf = pd.DataFrame(data=d)\ndf = df.sort_values(by='id', ignore_index=True)\ndf.to_csv('./submission.csv', index=False)\n    ","metadata":{},"execution_count":null,"outputs":[]}]}