{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"id":"HlaIywn-Qco0","execution":{"iopub.status.busy":"2021-06-21T13:54:28.011401Z","iopub.execute_input":"2021-06-21T13:54:28.011724Z","iopub.status.idle":"2021-06-21T13:54:28.022104Z","shell.execute_reply.started":"2021-06-21T13:54:28.011695Z","shell.execute_reply":"2021-06-21T13:54:28.02097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\nimport torch.utils.data as data_utils\n\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nimport numpy as np\nimport pandas as pd\n\nfrom pathlib import Path\nimport zipfile","metadata":{"execution":{"iopub.status.busy":"2021-06-21T13:54:28.024032Z","iopub.execute_input":"2021-06-21T13:54:28.024657Z","iopub.status.idle":"2021-06-21T13:54:28.032154Z","shell.execute_reply.started":"2021-06-21T13:54:28.024618Z","shell.execute_reply":"2021-06-21T13:54:28.031383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLEAN_IMGS = True  # clean extracted images at the end, to prevent overcrowding the output console\nCAT, DOG = 0, 1\ndef lbl_code_to_class(label):\n    return 'dog' if label == DOG else 'cat'\n\ndef class_to_lbl_code(class_name):\n    return CAT if class_name == 'cat' else DOG","metadata":{"execution":{"iopub.status.busy":"2021-06-21T13:54:28.035786Z","iopub.execute_input":"2021-06-21T13:54:28.036053Z","iopub.status.idle":"2021-06-21T13:54:28.043741Z","shell.execute_reply.started":"2021-06-21T13:54:28.036021Z","shell.execute_reply":"2021-06-21T13:54:28.04286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use GPU if available\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'Running on {device}')","metadata":{"id":"W-rTWyS-6ZzI","outputId":"882fb700-2ba5-4460-d397-dc492dbf4de5","execution":{"iopub.status.busy":"2021-06-21T13:54:28.044811Z","iopub.execute_input":"2021-06-21T13:54:28.045071Z","iopub.status.idle":"2021-06-21T13:54:28.05801Z","shell.execute_reply.started":"2021-06-21T13:54:28.045048Z","shell.execute_reply":"2021-06-21T13:54:28.056968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the dataset","metadata":{"id":"Oztht7ryQsOt"}},{"cell_type":"code","source":"# Extract and load data\n\nDATA_DIR = Path('/kaggle/working/data')\n\nif not all ([os.path.exists(DATA_DIR / directory) for directory in ['train', 'test']]):\n    print('Extracting data...')\n    if not os.path.exists(DATA_DIR / 'train'):\n        with zipfile.ZipFile(\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip\", 'r') as z:\n            z.extractall(DATA_DIR)\n    if not os.path.exists(DATA_DIR / 'test'):\n        with zipfile.ZipFile(\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip\", 'r') as z:\n            z.extractall(DATA_DIR)\n    print('Done')\n\nprint([str(DATA_DIR / directory) for directory in os.listdir(DATA_DIR)])\n# the zip files are extracted to new folders 'train' & 'test'\nTRAINING_DIR = DATA_DIR / 'train'\nTEST_DIR = DATA_DIR / 'test'\n\ntraining_set_full_raw = [TRAINING_DIR / filename for filename in os.listdir(TRAINING_DIR) if filename.endswith('.jpg')]\ntest_set_raw = [TEST_DIR / filename for filename in os.listdir(TEST_DIR) if filename.endswith('.jpg')]","metadata":{"execution":{"iopub.status.busy":"2021-06-21T13:54:28.059494Z","iopub.execute_input":"2021-06-21T13:54:28.0598Z","iopub.status.idle":"2021-06-21T13:54:30.757938Z","shell.execute_reply.started":"2021-06-21T13:54:28.059769Z","shell.execute_reply":"2021-06-21T13:54:30.755571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Peek at the data","metadata":{"id":"oA53K3TG7VC8"}},{"cell_type":"code","source":"SAMPLE_IMG_ID = np.random.choice(min(len(test_set_raw), len(training_set_full_raw)))\n\nprint(f'Training data: {len(training_set_full_raw)}, Sample: {training_set_full_raw[SAMPLE_IMG_ID]}')\nprint(f'Test data: {len(test_set_raw)}, Sample: {test_set_raw[SAMPLE_IMG_ID]}')\n\nsample_img, sample_img_label = Image.open(training_set_full_raw[SAMPLE_IMG_ID]), training_set_full_raw[SAMPLE_IMG_ID].name.split('.')[0]\nplt.axis(False)\njunk = plt.imshow(sample_img)\njunk = plt.title(sample_img_label)","metadata":{"id":"T_-lB4ap7WgQ","outputId":"7623c500-0971-4ebe-be69-d9dcccb56b7e","execution":{"iopub.status.busy":"2021-06-21T13:54:30.759257Z","iopub.status.idle":"2021-06-21T13:54:30.759702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Count and organize data","metadata":{}},{"cell_type":"code","source":"from collections import defaultdict\n\n\ntraining_full_classes = defaultdict(int)\n\nfor path in training_set_full_raw:\n    training_full_classes[path.name.split('.')[0]] += 1\n\nprint(f'Classes: {[(k,v) for k,v in training_full_classes.items()]}')\n\nNUM_CLASSES = len(training_full_classes.keys())\n\n# => 12500 / 12500 , data already balanced","metadata":{"execution":{"iopub.status.busy":"2021-06-21T13:54:30.761178Z","iopub.status.idle":"2021-06-21T13:54:30.76187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert to pytorch Dataset","metadata":{}},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, data, train=False, transformer=None):\n        self.data = data\n        self.train = train\n        self.transformer = transformer\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        img = Image.open(self.data[idx])\n        if self.transformer:\n            img = self.transformer(img)\n            \n        lbl = self.data[idx].name.split('.')[0]\n        lbl = class_to_lbl_code(lbl) if self.train else int(lbl)\n        return img.numpy().astype('float32'), lbl\n\ndata_transformer = transforms.Compose([\n#     transforms.Grayscale(num_output_channels=1),\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n#     transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))  # ImageNet parameters\n])\n\nnp.random.shuffle(training_set_full_raw)\ntraining_set_full = CustomDataset(training_set_full_raw, train=True, transformer=data_transformer)\ntest_set = CustomDataset(test_set_raw, train=False, transformer=data_transformer)\n\nIMAGE_SHAPE = training_set_full[0][0].shape\nprint(f'Image shape: {IMAGE_SHAPE}')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T13:54:30.763453Z","iopub.status.idle":"2021-06-21T13:54:30.764012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split to train / validation / test","metadata":{"id":"WEnwZ72AvPFs"}},{"cell_type":"code","source":"VALIDATION_FRAC = 0.1\nBATCH_SIZE = 64\n\nnp.random.seed(42)\nshuffled_idx = np.random.choice(len(training_set_full), len(training_set_full), replace=False)\nvalidation_set = data_utils.Subset(training_set_full, shuffled_idx[:int(len(training_set_full) * 0.1)])\ntraining_set = data_utils.Subset(training_set_full, shuffled_idx[int(len(training_set_full) * 0.1):])\n\nprint('Data size after splitting to train / validation / test:')\nprint(f'Training data: {len(training_set)}, Sample structure: ({type(training_set[0][0])} {training_set[0][0].shape}, {type(training_set[0][1])})')\nprint(f'Validation data: {len(validation_set)}, Sample structure: ({type(validation_set[0][0])} {validation_set[0][0].shape}, {type(validation_set[0][1])})')\nprint(f'Test data: {len(test_set)}, Sample structure: ({type(test_set[0][0])} {test_set[0][0].shape}, {type(test_set[0][1])})')\n\ntraining_loader = DataLoader(dataset=training_set, batch_size=BATCH_SIZE, shuffle=True)\ntest_loader = DataLoader(dataset=test_set, batch_size=BATCH_SIZE, shuffle=False)\nvalidation_loader = DataLoader(dataset=validation_set, batch_size=BATCH_SIZE, shuffle=False)  # don't shuffle (!) to visualize results correctly.\n                                                                                              # it's ok because we shuffled the original ds.","metadata":{"id":"DXicX97wvNvm","outputId":"cfc48690-72f3-41ca-e516-60434523a116","execution":{"iopub.status.busy":"2021-06-21T13:54:30.765276Z","iopub.status.idle":"2021-06-21T13:54:30.765829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"AwBijfbSSkvE"}},{"cell_type":"code","source":"def calc_out_size(in_size, padding, kernel, stride):\n    # formula from: https://youtu.be/wnK3uWv_WkU?t=234\n    # assuming nxn\n    padding = padding if isinstance(padding, int) else padding[0]\n    kernel = kernel if isinstance(kernel, int) else kernel[0]\n    stride = stride if isinstance(stride, int) else stride[0]\n    return ((in_size + 2*padding - kernel) // stride) + 1\n\n\nclass CNN(nn.Module):\n    def __init__(self, dimensions, in_channels, num_classes):\n        super().__init__()\n        self.dimensions = dimensions\n        self.num_classes = num_classes\n        self.current_size = 0\n        \n        # pooling\n        self.pool = nn.MaxPool2d(kernel_size=3)\n        \n        # dropout\n        self.dropout = nn.Dropout(0.5)\n                \n        # layer 1\n        self.conv1 = nn.Sequential(nn.Conv2d(in_channels=in_channels, out_channels=32, kernel_size=(3,3)),)\n#                                    nn.BatchNorm2d(32))  # Makes loss unstable!!\n        self.update_size(self.conv1[0], dims=self.dimensions)\n        self.update_size(self.pool)\n        \n        # layer 2\n        self.conv2 = nn.Sequential(nn.Conv2d(in_channels=self.conv1[0].out_channels, out_channels=64, kernel_size=(3,3)),)\n#                                    nn.BatchNorm2d(64))\n        self.update_size(self.conv2[0])\n        self.update_size(self.pool)\n        \n        # layer 3\n        self.conv3 = nn.Sequential(nn.Conv2d(in_channels=self.conv2[0].out_channels, out_channels=128, kernel_size=(3,3)),)\n#                                    nn.BatchNorm2d(128))\n        self.update_size(self.conv3[0])\n        self.update_size(self.pool)\n        \n        # layer 4\n#         self.conv4 = nn.Sequential(nn.Conv2d(in_channels=self.conv3[0].out_channels, out_channels=256, kernel_size=(3,3)),)\n# #                                    nn.BatchNorm2d(256))\n#         self.update_size(self.conv4[0])\n#         self.update_size(self.pool)\n    \n        # layer 5\n        self.fc1 = nn.Linear(self.conv3[0].out_channels*self.current_size*self.current_size, 512)\n        \n        # layer 6\n        self.fc2 = nn.Linear(512, self.num_classes)\n        \n\n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n        x = self.pool(x)\n        \n        x = F.relu(self.conv2(x))\n        x = self.pool(x)\n\n        x = F.relu(self.conv3(x))\n        x = self.pool(x)\n        \n#         x = F.relu(self.conv4(x))\n#         x = self.pool(x)\n        \n        x = x.reshape(x.shape[0], -1)\n        x = F.relu(self.fc1(x))\n        x = self.dropout(x)\n        \n        x = self.fc2(x)\n        # TODO: softmax?\n        \n        return x\n    \n    def update_size(self, layer, dims=None):\n        # assuming width == height\n        self.current_size = calc_out_size(in_size=dims if dims else self.current_size, padding=layer.padding, kernel=layer.kernel_size, stride=layer.stride)\n        print(f'self.current_size = {self.current_size}')\n\nmodel = CNN(IMAGE_SHAPE[-1], IMAGE_SHAPE[0], NUM_CLASSES)\nmodel.to(device)\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nloss_function = nn.CrossEntropyLoss()","metadata":{"id":"ZCI8UvXzSi0C","execution":{"iopub.status.busy":"2021-06-21T13:54:30.767436Z","iopub.status.idle":"2021-06-21T13:54:30.768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2021-06-21T13:54:30.769248Z","iopub.status.idle":"2021-06-21T13:54:30.769786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, loss_function, X, y):\n    predictions = model(X)\n    \n    loss = loss_function(predictions, y)\n    predictions = predictions.argmax(dim=1).cpu().numpy()\n    acc = (predictions == y.cpu().numpy()).mean()\n    return predictions, acc, loss","metadata":{"id":"8hXqcCoH9X2q","execution":{"iopub.status.busy":"2021-06-21T13:54:30.77127Z","iopub.status.idle":"2021-06-21T13:54:30.771807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{"id":"13tpzn3_HBCM"}},{"cell_type":"code","source":"EPOCHS = 7\nEVALUATION_FREQ = len(training_set) // BATCH_SIZE // 10  # guarantee 10 evaluations per epoch\nEARLY_STOPPING_K = 2  # consecutive non-increase to stop (None or 0 to not stop early)\nEARLY_STOPPING_EPS = 0.01\n\nmodel.train(mode=True)  # just puts the model in training mode (doesn't actually train)\n\ntraining_acc_lst, training_loss_lst = [], []\nvalidation_acc_lst, validation_loss_lst = [], []\nnot_increasing = 0\nfor epoch in range(EPOCHS):\n    print(f'Epoch {epoch + 1}')\n    prev_epoch_acc, epoch_acc = 0, 0\n    training_acc_checkpoint, training_loss_checkpoint = [], []\n    for batch_idx, (data, labels) in enumerate(training_loader):\n        # cast to device (cpu / gpu) (gpu for faster op). also both need to be on same device.\n        data, labels = data.to(device), labels.to(device)\n        \n        # run model on data to get predictions.\n        # data is of shape (BATCH_SIZE, 1, 224, 224),\n        predictions, acc, loss = evaluate(model, loss_function, data, labels)\n        training_acc_checkpoint.append(acc)\n\n        # loss already calculated in the evaluate() call. just append it\n        training_loss_checkpoint.append(loss.item())\n        \n        # back propagation (calculate the gradient)\n        loss.backward()\n\n        # gradient descent (adjust the weights)\n        optimizer.step()\n\n        # default behavior of pytorch is to NOT clear the gradients after every step.\n        # but we need to clear them to prevent accumulation of gradients throughout iterations.\n        optimizer.zero_grad()  # or model.zero_grad() if all the model's parameters are in the optimizer (in our case they are)\n\n        # evaluate on validation\n        if batch_idx % EVALUATION_FREQ == 0 or batch_idx == len(training_loader) - 1:\n            # average training acc and loss every EVALUATION_FREQ, so our training and validation plots axes will have the same length\n            training_acc_lst.append(np.mean(training_acc_checkpoint))\n            training_loss_lst.append(np.mean(training_loss_checkpoint))\n            # restart checkpoints\n            training_acc_checkpoint, training_loss_checkpoint = [], []\n\n            # predict validation data, but first disable gradient tracking, and enter evaluation mode\n            model.train(mode=False)  # enter eval mode. suggested here: https://stackoverflow.com/a/55627781/900394\n            with torch.no_grad():  # locally disable gradient tracking\n                validation_acc_checkpoint, validation_loss_checkpoint = [], []\n                validation_predictions = []  # saved for showing results later\n                for val_batch_idx, (val_data, val_labels) in enumerate(validation_loader):\n                    val_data, val_labels = val_data.to(device), val_labels.to(device)\n\n                    val_predictions, validation_acc, validation_loss = evaluate(model, loss_function, val_data, val_labels)\n                    \n                    validation_loss_checkpoint.append(validation_loss.item())\n                    validation_acc_checkpoint.append(validation_acc)\n                    validation_predictions.extend(val_predictions)  # predictions are for a complete batch, so we need to \"extend\" not \"append\"\n                \n                validation_acc_lst.append(np.mean(validation_acc_checkpoint))\n                validation_loss_lst.append(np.mean(validation_loss_checkpoint))\n                \n                if batch_idx == len(training_loader) - 1:\n                    prev_epoch_acc = epoch_acc\n                    epoch_acc = validation_acc_lst[-1]\n            \n            print(f'Training acc: {training_acc_lst[-1]:.2f}, Training loss: {training_loss_lst[-1]:.2f}, Validation acc: {validation_acc_lst[-1]:.2f}, Validation loss: {validation_loss_lst[-1]:.2f}')\n\n            model.train(mode=True)  # re-enter training mode\n\n    # epoch end\n    \n    # early stopping according to validation accuracy\n    increase = epoch_acc - prev_epoch_acc\n    not_increasing = (not_increasing + 1) if increase < EARLY_STOPPING_EPS else 0\n    if EARLY_STOPPING_K and not_increasing >= EARLY_STOPPING_K:\n        print(f'Less than {EARLY_STOPPING_EPS} accuracy increase in the last {not_increasing} consecutive epochs. Early stopping...')\n        break\n\nprint('\\nDone training.')\n\njunk = model.train(mode=False)  # exit training mode","metadata":{"id":"vBhWogA_HAVA","outputId":"dd992b51-20bf-42c9-915c-eab08ca3db38","execution":{"iopub.status.busy":"2021-06-21T13:54:30.773248Z","iopub.status.idle":"2021-06-21T13:54:30.773788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot results","metadata":{"id":"q2KnqE8pWYP1"}},{"cell_type":"code","source":"plot_checkpoints = (0, None)  # 'None' to plot to the end\nplt.figure(figsize=(30, 10))\n        \n# accuracy\nplt.subplot(2,4,1)\nplt.ylim(0, 1.1)\nplt.plot(range(len(validation_acc_lst[plot_checkpoints[0]:plot_checkpoints[1]])), validation_acc_lst[plot_checkpoints[0]:plot_checkpoints[1]])\nplt.plot(range(len(training_acc_lst[plot_checkpoints[0]:plot_checkpoints[1]])), training_acc_lst[plot_checkpoints[0]:plot_checkpoints[1]])\nplt.legend(['val acc', 'training acc'])\n\n# loss\nplt.subplot(2,4,2)\nplt.ylim(0, 1.1)\nplt.plot(range(len(validation_loss_lst[plot_checkpoints[0]:plot_checkpoints[1]])), validation_loss_lst[plot_checkpoints[0]:plot_checkpoints[1]])\nplt.plot(range(len(training_loss_lst[plot_checkpoints[0]:plot_checkpoints[1]])), training_loss_lst[plot_checkpoints[0]:plot_checkpoints[1]])\nplt.legend(['val loss', 'training loss'])\n\nplt.show()","metadata":{"id":"_Nvk9wGyWV1v","outputId":"09ec55dd-feb0-48f2-8d89-c39723ed3813","execution":{"iopub.status.busy":"2021-06-21T13:54:30.77528Z","iopub.status.idle":"2021-06-21T13:54:30.775819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize validation results","metadata":{"id":"3rmXrJdRXvp6"}},{"cell_type":"code","source":"DRAW_IMGS = 10\n\n# visualize validation results\nfig = plt.figure(figsize=(8, 5))\nfig.tight_layout()\nshow_imgs_idx = np.random.choice(len(validation_set), DRAW_IMGS, replace=False)\nfor i, (val_test_sample, val_test_label) in enumerate(data_utils.Subset(validation_set, show_imgs_idx)):\n    plt.subplot(2, DRAW_IMGS // 2, i+1)\n    plt.title(f'Actual: {lbl_code_to_class(val_test_label)}\\nPredicted: {lbl_code_to_class(validation_predictions[show_imgs_idx[i]])}')\n    plt.axis('off')\n    plt.imshow(val_test_sample[0,::], cmap='gray')","metadata":{"id":"idkx8dhJXu3V","outputId":"c4d92043-a1f1-4a82-899e-6668503e543c","execution":{"iopub.status.busy":"2021-06-21T13:54:30.777115Z","iopub.status.idle":"2021-06-21T13:54:30.777657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing","metadata":{"id":"Bp5jSTErlJXS"}},{"cell_type":"code","source":"model.train(False)  # ensure we're in eval mode\n\ntest_predictions = []\ntest_idx = []\nprint('Testing...')\nfor X,id_ in test_loader:\n    with torch.no_grad():\n        X = X.to(device)\n        predictions = model(X)\n\n        test_idx.extend(id_.cpu().numpy())\n        test_predictions.extend(F.softmax(predictions, dim=1)[:, 1].cpu().numpy())  # they want the probability that the sample is a dog:\n                                                      # https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/overview/evaluation#:~:text=probability%20that%20image%20is%20a%20dog\n\nsubmission = pd.DataFrame({'id': test_idx, 'label': test_predictions}).sort_values(by='id')\nsubmission.to_csv(f'submission.csv', index=False)\nprint(f'Submission saved')","metadata":{"id":"6JDQIxGElIn_","outputId":"4e9b2fb3-4b56-4a86-f4e6-4430d122c61c","execution":{"iopub.status.busy":"2021-06-21T13:54:30.778929Z","iopub.status.idle":"2021-06-21T13:54:30.779484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CLEAN_IMGS:\n    from shutil import rmtree\n    from os import path\n    if path.exists(TRAINING_DIR):\n        rmtree(TRAINING_DIR)\n    if path.exists(TEST_DIR):\n        rmtree(TEST_DIR)\n    print('Images removed.')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T13:54:30.78073Z","iopub.status.idle":"2021-06-21T13:54:30.781297Z"},"trusted":true},"execution_count":null,"outputs":[]}]}