{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-01T19:20:00.648009Z","iopub.execute_input":"2022-01-01T19:20:00.648272Z","iopub.status.idle":"2022-01-01T19:20:01.426055Z","shell.execute_reply.started":"2022-01-01T19:20:00.648243Z","shell.execute_reply":"2022-01-01T19:20:01.425329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchinfo","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:01.428865Z","iopub.execute_input":"2022-01-01T19:20:01.429122Z","iopub.status.idle":"2022-01-01T19:20:09.250636Z","shell.execute_reply.started":"2022-01-01T19:20:01.429086Z","shell.execute_reply":"2022-01-01T19:20:09.249899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\n\nimport wandb\n\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb\")\n\nwandb.init(project='Cat-vs-Dog-CNN', save_code=True)\n\n# https://github.com/TylerYep/torchinfo\nfrom torchinfo import summary # conda install -c conda-forge torchinfo","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:09.252591Z","iopub.execute_input":"2022-01-01T19:20:09.253015Z","iopub.status.idle":"2022-01-01T19:20:25.807131Z","shell.execute_reply.started":"2022-01-01T19:20:09.252975Z","shell.execute_reply":"2022-01-01T19:20:25.806383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"train_dir = 'train'\ntest_dir = 'test'\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as train_zip:\n    train_zip.extractall('')\n    \nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as test_zip:\n    test_zip.extractall('')\ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\nprint(f\"Train Data: {len(train_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:25.808822Z","iopub.execute_input":"2022-01-01T19:20:25.809082Z","iopub.status.idle":"2022-01-01T19:20:40.593527Z","shell.execute_reply.started":"2022-01-01T19:20:25.809045Z","shell.execute_reply":"2022-01-01T19:20:40.592747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [path.split('/')[-1].split('.')[0] for path in train_list]","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:40.595845Z","iopub.execute_input":"2022-01-01T19:20:40.596417Z","iopub.status.idle":"2022-01-01T19:20:40.613164Z","shell.execute_reply.started":"2022-01-01T19:20:40.596378Z","shell.execute_reply":"2022-01-01T19:20:40.612455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot random image with their label","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1, len(train_list), size=9)\nfig, axes = plt.subplots(3, 3, figsize=(16, 12))\n\nfor idx, ax in enumerate(axes.ravel()):\n    img = Image.open(train_list[idx])\n    ax.set_title(labels[idx])\n    ax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:40.614188Z","iopub.execute_input":"2022-01-01T19:20:40.614697Z","iopub.status.idle":"2022-01-01T19:20:42.431042Z","shell.execute_reply.started":"2022-01-01T19:20:40.614640Z","shell.execute_reply":"2022-01-01T19:20:42.429618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Use Sklearn to split data","metadata":{}},{"cell_type":"code","source":"# we reserve 20% of the training set for the validation and the remaining 80% for training\nVALIDATION_RATIO = 0.2\n\ntrain_list, valid_list = train_test_split(train_list, \n                                          test_size=VALIDATION_RATIO,\n                                          stratify=labels,\n                                          random_state=0)\n\ntraining_size = len(train_list)\nvalidation_size = len(valid_list)\ntesting_size = len(test_list)\n\nprint(f\"Train Data: {training_size}\")\nprint(f\"Validation Data: {validation_size}\")\nprint(f\"Test Data: {testing_size}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.432367Z","iopub.execute_input":"2022-01-01T19:20:42.432792Z","iopub.status.idle":"2022-01-01T19:20:42.477026Z","shell.execute_reply.started":"2022-01-01T19:20:42.432756Z","shell.execute_reply":"2022-01-01T19:20:42.476219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will discuss this in more detail in a near future...","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n#         transforms.Resize(128), # makes it easier for the GPU\n        transforms.Resize((227, 227)),\n#         transforms.RandomResizedCrop(112),\n#         transforms.RandomResizedCrop(227, 227),\n#         transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n\nval_transforms = transforms.Compose([\n#         transforms.Resize(128),\n        transforms.Resize((227, 227)),\n#         transforms.CenterCrop(112),\n#         transforms.CenterCrop(227, 227),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n\n\ntest_transforms = transforms.Compose([\n#         transforms.Resize(128),\n        transforms.Resize((227, 227)),\n#         transforms.CenterCrop(112),\n#         transforms.CenterCrop(227, 227),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.478213Z","iopub.execute_input":"2022-01-01T19:20:42.478728Z","iopub.status.idle":"2022-01-01T19:20:42.486174Z","shell.execute_reply.started":"2022-01-01T19:20:42.478689Z","shell.execute_reply":"2022-01-01T19:20:42.485250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the dataset using PIL to read image","metadata":{}},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n        self.filelength = len(file_list)\n\n    def __len__(self):\n        return self.filelength\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        img_transformed = self.transform(img)\n        label = img_path.split(\"/\")[-1].split(\".\")[0]\n        label = 1 if label == \"dog\" else 0\n        return img_transformed, label","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.487331Z","iopub.execute_input":"2022-01-01T19:20:42.487838Z","iopub.status.idle":"2022-01-01T19:20:42.498917Z","shell.execute_reply.started":"2022-01-01T19:20:42.487801Z","shell.execute_reply":"2022-01-01T19:20:42.498083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = CatsDogsDataset(train_list, transform=train_transforms)\nvalid_data = CatsDogsDataset(valid_list, transform=test_transforms)\ntest_data = CatsDogsDataset(test_list, transform=test_transforms)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.500236Z","iopub.execute_input":"2022-01-01T19:20:42.500583Z","iopub.status.idle":"2022-01-01T19:20:42.508056Z","shell.execute_reply.started":"2022-01-01T19:20:42.500547Z","shell.execute_reply":"2022-01-01T19:20:42.507313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create dataloader, you can modify the batch size if needed","metadata":{}},{"cell_type":"code","source":"batch_size = 32\ntrain_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(dataset=valid_data, batch_size=batch_size, shuffle=False)\ntest_loader = DataLoader(dataset=test_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.509792Z","iopub.execute_input":"2022-01-01T19:20:42.510058Z","iopub.status.idle":"2022-01-01T19:20:42.519120Z","shell.execute_reply.started":"2022-01-01T19:20:42.510014Z","shell.execute_reply":"2022-01-01T19:20:42.518315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataiter = iter(train_loader)\nimages, labels = dataiter.next()\n\nprint(images.shape) # (number_of_images, batch, image_width, image_height)\nprint(torch.unique(labels).size(dim=0))","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.520398Z","iopub.execute_input":"2022-01-01T19:20:42.520755Z","iopub.status.idle":"2022-01-01T19:20:42.670585Z","shell.execute_reply.started":"2022-01-01T19:20:42.520720Z","shell.execute_reply":"2022-01-01T19:20:42.669073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(images[0].permute(1, 2, 0).cpu().squeeze())\nplt.title(\"Ground Truth: {}\".format(labels[0]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.671835Z","iopub.execute_input":"2022-01-01T19:20:42.672080Z","iopub.status.idle":"2022-01-01T19:20:42.862253Z","shell.execute_reply.started":"2022-01-01T19:20:42.672045Z","shell.execute_reply":"2022-01-01T19:20:42.861530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convolutional Neural Network","metadata":{}},{"cell_type":"code","source":"# If you think 224x224 is the right input size for AlexNet please\n# @see see https://cs231n.github.io/convolutional-networks/\n# @see https://stackoverflow.com/questions/36733636/number-of-neurons-in-alexnet\n# @see https://datascience.stackexchange.com/questions/29245/what-is-the-input-size-of-alex-net\n# @see https://learnopencv.com/understanding-alexnet/\nclass AlexNet(nn.Module):\n    def __init__(self, num_classes: int):\n        super(AlexNet, self).__init__()\n        \n        #----------------------------\n        # CONVOLUTIONAL LAYERS\n        #----------------------------\n        self.feature_extraction = nn.Sequential(\n            # 1st Convolutional Layer\n            nn.Conv2d(in_channels=3, out_channels=96, kernel_size=11, stride=4, padding=2, bias=False),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=3, stride=2, padding=0),\n            \n            # 2nd Convolutional Layer\n            nn.Conv2d(in_channels=96, out_channels=192, kernel_size=5, stride=1, padding=2, bias=False),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=3, stride=2, padding=0),\n            \n            # 3rd Convolutional Layer\n            nn.Conv2d(in_channels=192, out_channels=384, kernel_size=3, stride=1, padding=1, bias=False),\n            nn.ReLU(inplace=True),\n\n            # 4th Convolutional Layer\n            nn.Conv2d(in_channels=384, out_channels=256, kernel_size=3, stride=1, padding=1, bias=False),\n            nn.ReLU(inplace=True),\n            \n            # 5th Convolutional Layer\n            nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=1, bias=False),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=3, stride=2, padding=0),\n        )\n\n        #----------------------------\n        # FULLY CONNECTED LAYERS\n        #----------------------------\n        self.classifier = nn.Sequential(\n            # 1st Fully Connected Layer\n            nn.Linear(in_features=(256 * 6 * 6), out_features=4096),\n            nn.ReLU(inplace=True),\n            # Dropout to prevent overfitting\n            nn.Dropout(p=0.5),\n\n            # 2nd Fully Connected Layer\n            nn.Linear(in_features=4096, out_features=4096),\n            nn.ReLU(inplace=True),\n            # Dropout to prevent overfitting\n            nn.Dropout(p=0.5),\n\n            # 3rd Fully Connected Layer\n            nn.Linear(in_features=4096, out_features=num_classes),\n        )\n\n    # Here we actually _build_ the net.    \n    def forward(self,x) -> torch.Tensor:\n        # Convolutional layers\n        x = self.feature_extraction(x)\n        # Flatten\n        x = x.view(-1, 256 * 6 * 6)\n        # Fully connected layers\n        x = self.classifier(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.865178Z","iopub.execute_input":"2022-01-01T19:20:42.865700Z","iopub.status.idle":"2022-01-01T19:20:42.879194Z","shell.execute_reply.started":"2022-01-01T19:20:42.865646Z","shell.execute_reply":"2022-01-01T19:20:42.878318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = AlexNet(2)\n# print(model)\n\nsummary(model, input_size=(batch_size, 3, 227, 227))","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:42.895688Z","iopub.execute_input":"2022-01-01T19:20:42.896626Z","iopub.status.idle":"2022-01-01T19:20:43.474332Z","shell.execute_reply.started":"2022-01-01T19:20:42.896587Z","shell.execute_reply":"2022-01-01T19:20:43.473543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run NN on CPU or GPU?\nInit the model and put it on GPU","metadata":{}},{"cell_type":"code","source":"# cuda:0, in case of multiple GPUs we will use the first one (0)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint('Detected device: {}'.format(device))\n# Put model to device\nmodel.to(device)\n# batch.to(device)\nprint('Training on {}!'.format(device))","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:43.475587Z","iopub.execute_input":"2022-01-01T19:20:43.476050Z","iopub.status.idle":"2022-01-01T19:20:43.483064Z","shell.execute_reply.started":"2022-01-01T19:20:43.476010Z","shell.execute_reply":"2022-01-01T19:20:43.482256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defines the learning rates for the parameter updates\nlr_rate = 1e-3 # e.q to 0.003, you can change it if needed\n# To update the hyperparameters of the model.\noptimizer = torch.optim.SGD(model.parameters(), lr=lr_rate) # momentum=0.5 decreased the accuracy of almost 2%\ncriterion = nn.CrossEntropyLoss()\n\nEPOCHS = 50\n\nfor epoch in range(EPOCHS):\n    # TRAINING LOOP\n    training_loss = 0\n    for images, labels in train_loader:\n        images = images.to(device)\n        labels = labels.to(device)\n        # -------------\n        # Forward Pass\n        # -------------\n        # Clear the gradients as we don't want any gradient from previous epoch\n        # to carry forward: don't want to cummulate gradients.\n        optimizer.zero_grad()\n        # Forward Pass\n        output = model(images)\n        # Find the Loss\n        loss = criterion(output, labels)\n        # Calculate gradients\n        loss.backward()\n        # Update Weights\n        optimizer.step()\n        training_loss += loss.item()\n    \n    # VALIDATION LOOP\n    with torch.no_grad(): # we don't need gradients in the validation phase\n        validation_loss = 0\n        correct_classified = 0\n        for images, labels in valid_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n            # -------------\n            # Forward Pass\n            # -------------\n            output = model(images)\n            # Returns the maximum value of all elements in the input tensor (predictions).\n            # dim: the dimension to reduce the tensor to.\n            # _: prediction confidence\n            # predicted: prediction label\n            _, predicted = torch.max(output, dim=1)\n            # Find the Loss\n            loss = criterion(output, labels)\n            # Calculate Loss\n            validation_loss += loss.item()\n            \n            # Comparing the prediction (predicted) with the ground truth (labels)\n            correct_classified += int(predicted.eq(labels).sum().item())\n\n    validation_accuracy = correct_classified / validation_size\n    print('Epoch {}'.format(epoch+1),\n          \"\\t training_loss: \", training_loss,\n          '\\t validation_loss: ', validation_loss,\n          '\\t validation_accuracy: ', validation_accuracy),\n    wandb.log({'training_loss': training_loss, 'validation_loss': validation_loss, 'validation_accuracy': validation_accuracy})","metadata":{"execution":{"iopub.status.busy":"2022-01-01T19:20:43.484470Z","iopub.execute_input":"2022-01-01T19:20:43.484986Z","iopub.status.idle":"2022-01-01T21:01:51.584837Z","shell.execute_reply.started":"2022-01-01T19:20:43.484950Z","shell.execute_reply":"2022-01-01T21:01:51.584019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}