{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# Kaggle: \n# Wandb:\n# Best Score: 0.74429\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-14T15:22:06.668036Z","iopub.execute_input":"2024-01-14T15:22:06.669124Z","iopub.status.idle":"2024-01-14T15:22:07.224772Z","shell.execute_reply.started":"2024-01-14T15:22:06.669055Z","shell.execute_reply":"2024-01-14T15:22:07.223308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms\nimport wandb\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:22:07.227338Z","iopub.execute_input":"2024-01-14T15:22:07.228230Z","iopub.status.idle":"2024-01-14T15:22:14.671300Z","shell.execute_reply.started":"2024-01-14T15:22:07.228183Z","shell.execute_reply":"2024-01-14T15:22:14.669678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    'device': 'cuda' if torch.cuda.is_available() else 'cpu',\n    'lr': 1e-3,\n    'batch': 128,\n    'epochs': 40,\n    'loss_fn': 'BCELoss', # BCELoss, NLLLoss\n    'optimizer': 'Adam', # Adam, SGD\n    'momentum': 0.9\n}\n\nprint(config['device'])\n\nwandb.init(project='Dogs vs Cat', save_code=True, config=config, mode='disabled')","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:22:14.673021Z","iopub.execute_input":"2024-01-14T15:22:14.674206Z","iopub.status.idle":"2024-01-14T15:22:53.801115Z","shell.execute_reply.started":"2024-01-14T15:22:14.674132Z","shell.execute_reply":"2024-01-14T15:22:53.799767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"train_dir = 'train'\ntest_dir = 'test'\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as train_zip:\n    train_zip.extractall('')\n    \nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as test_zip:\n    test_zip.extractall('')\ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\nprint(f\"Train Data: {len(train_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:22:57.531998Z","iopub.execute_input":"2024-01-14T15:22:57.532541Z","iopub.status.idle":"2024-01-14T15:23:18.455198Z","shell.execute_reply.started":"2024-01-14T15:22:57.532500Z","shell.execute_reply":"2024-01-14T15:23:18.453816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [path.split('/')[-1].split('.')[0] for path in train_list]","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:18.457793Z","iopub.execute_input":"2024-01-14T15:23:18.459120Z","iopub.status.idle":"2024-01-14T15:23:18.875735Z","shell.execute_reply.started":"2024-01-14T15:23:18.459066Z","shell.execute_reply":"2024-01-14T15:23:18.874422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot random image with their label","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1, len(train_list), size=9)\nfig, axes = plt.subplots(3, 3, figsize=(16, 12))\n\nfor idx, ax in enumerate(axes.ravel()):\n    img = Image.open(train_list[idx])\n    ax.set_title(labels[idx])\n    ax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:18.877832Z","iopub.execute_input":"2024-01-14T15:23:18.881328Z","iopub.status.idle":"2024-01-14T15:23:22.678764Z","shell.execute_reply.started":"2024-01-14T15:23:18.881264Z","shell.execute_reply":"2024-01-14T15:23:22.677770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Use Sklearn to split the training data","metadata":{}},{"cell_type":"code","source":"train_list, valid_list = train_test_split(train_list, \n                                          test_size=0.2,\n                                          stratify=labels,\n                                          random_state=0)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:22.681410Z","iopub.execute_input":"2024-01-14T15:23:22.681934Z","iopub.status.idle":"2024-01-14T15:23:23.153411Z","shell.execute_reply.started":"2024-01-14T15:23:22.681894Z","shell.execute_reply":"2024-01-14T15:23:23.152265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train Data: {len(train_list)}\")\nprint(f\"Validation Data: {len(valid_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:23.155114Z","iopub.execute_input":"2024-01-14T15:23:23.155722Z","iopub.status.idle":"2024-01-14T15:23:23.599514Z","shell.execute_reply.started":"2024-01-14T15:23:23.155667Z","shell.execute_reply":"2024-01-14T15:23:23.598153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will discuss this in more detail in a near future...","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomRotation((-90, 90)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # RESNET\n])\n\ntest_transforms = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:23.601801Z","iopub.execute_input":"2024-01-14T15:23:23.602221Z","iopub.status.idle":"2024-01-14T15:23:25.164374Z","shell.execute_reply.started":"2024-01-14T15:23:23.602183Z","shell.execute_reply":"2024-01-14T15:23:25.162775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the dataset using PIL to read image","metadata":{}},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n        self.filelength = len(file_list)\n\n    def __len__(self):\n        return self.filelength\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path).convert('RGB')\n        img_transformed = self.transform(img)\n        label = img_path.split(\"/\")[-1].split(\".\")[0]\n        label = 1 if label == \"dog\" else 0\n        \n        l = [0,0]\n        l[label] = 1\n        \n        return img_transformed, torch.tensor(l).float()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:25.166206Z","iopub.execute_input":"2024-01-14T15:23:25.167087Z","iopub.status.idle":"2024-01-14T15:23:26.182454Z","shell.execute_reply.started":"2024-01-14T15:23:25.167029Z","shell.execute_reply":"2024-01-14T15:23:26.181252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = CatsDogsDataset(train_list, transform=train_transforms)\nvalid_data = CatsDogsDataset(valid_list, transform=test_transforms)\ntest_data = CatsDogsDataset(test_list, transform=test_transforms)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:26.184058Z","iopub.execute_input":"2024-01-14T15:23:26.184846Z","iopub.status.idle":"2024-01-14T15:23:26.679277Z","shell.execute_reply.started":"2024-01-14T15:23:26.184783Z","shell.execute_reply":"2024-01-14T15:23:26.677890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create dataloader, you can modify the batch size if needed","metadata":{}},{"cell_type":"code","source":"train_loader = DataLoader(dataset=train_data, batch_size=config['batch'], shuffle=True)\nvalid_loader = DataLoader(dataset=valid_data, batch_size=config['batch'], shuffle=False)\ntest_loader = DataLoader(dataset=test_data, batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:26.680834Z","iopub.execute_input":"2024-01-14T15:23:26.681958Z","iopub.status.idle":"2024-01-14T15:23:27.119046Z","shell.execute_reply.started":"2024-01-14T15:23:26.681900Z","shell.execute_reply":"2024-01-14T15:23:27.117792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO: define a convolutional neural network, train it and output the log \n# from efficientnet \ndef _weights_init(m):\n    if isinstance(m, nn.Conv2d):\n        nn.init.kaiming_normal_(m.weight, mode=\"fan_out\")\n        if m.bias is not None:\n            nn.init.zeros_(m.bias)\n    elif isinstance(m, nn.BatchNorm2d):\n        nn.init.ones_(m.weight)\n        nn.init.zeros_(m.bias)\n    elif isinstance(m, nn.Linear):\n        nn.init.xavier_normal_(m.weight)\n        nn.init.constant_(m.bias, 0)\n    return m\n\ndef create_layer(c_in, c_out, kernel, stride=2, padding=1):\n    return nn.Sequential(\n            _weights_init(nn.Conv2d(c_in, c_out, kernel_size=kernel, stride=stride, padding=padding)),\n            _weights_init(nn.BatchNorm2d(c_out)),\n            nn.ReLU(),\n            nn.MaxPool2d(2),\n        )\n\nclass Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        \n        self.layer1 = create_layer(3, 16, 5)\n        self.layer2 = create_layer(16, 32, 5)\n        self.layer3 = create_layer(32, 64, 3, stride=1)\n                \n        self.fc = nn.Sequential(\n            nn.Dropout(0.2),\n            _weights_init(nn.Linear(2304, 1024)),\n            nn.ReLU(),\n            _weights_init(nn.Linear(1024, 512)),\n            nn.ReLU(),\n            _weights_init(nn.Linear(512, 2)),\n        )\n        \n        self.log_softmax = nn.LogSoftmax(dim=1)\n        self.sigmoid = nn.Sigmoid()\n        \n    def forward(self, x):\n        out = self.layer1(x)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        \n        out = out.reshape(out.shape[0], -1)\n        out = self.fc(out)\n        \n        if config['loss_fn'] == 'NLLLoss':\n            out = self.log_softmax(out)\n        if config['loss_fn'] == 'BCELoss':\n            out = self.sigmoid(out)\n        \n        return out","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:27.123198Z","iopub.execute_input":"2024-01-14T15:23:27.124558Z","iopub.status.idle":"2024-01-14T15:23:27.525387Z","shell.execute_reply.started":"2024-01-14T15:23:27.124494Z","shell.execute_reply":"2024-01-14T15:23:27.524023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model().to(config['device'])\n\nif config['loss_fn'] == 'BCELoss':\n    loss_fn = nn.BCELoss()\nelif config['loss_fn'] == 'NLLLoss':\n    loss_fn = nn.NLLLoss()\nelse:\n    raise Exception('Invalid Loss function')\n\nif config['optimizer'] == 'Adam':\n    optimizer = optim.Adam(model.parameters(), lr=config['lr'])\nelif config['optimizer'] == 'SGD':\n    optimizer = optim.SGD(model.parameters(), lr=config['lr'], momentum=config['momentum'])\nelse:\n    raise Exception('Invalid Optimizer')","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:27.527913Z","iopub.execute_input":"2024-01-14T15:23:27.528430Z","iopub.status.idle":"2024-01-14T15:23:28.151758Z","shell.execute_reply.started":"2024-01-14T15:23:27.528379Z","shell.execute_reply":"2024-01-14T15:23:28.150382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loss = []\nval_loss = []\ntrain_accuracy = []\nval_accuracy = []","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:28.154213Z","iopub.execute_input":"2024-01-14T15:23:28.154884Z","iopub.status.idle":"2024-01-14T15:23:28.522355Z","shell.execute_reply.started":"2024-01-14T15:23:28.154829Z","shell.execute_reply":"2024-01-14T15:23:28.520918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_val_loss = 1e10\n\nfor e in range(config['epochs']): \n    \n    print(f\"{e+1}/{config['epochs']}\")\n    \n    model.train()\n    running_loss = 0\n    running_correct = 0\n    for image, label in train_loader:\n        image, label = image.to(config['device']), label.to(config['device'])\n        optimizer.zero_grad()\n        pred = model(image)\n        loss = loss_fn(pred, label)\n        loss.backward()\n        optimizer.step()\n        \n        running_correct += (pred.argmax(dim=1) == label.argmax(dim=1)).float().sum().item()\n        running_loss += loss.item()\n\n    train_loss.append(running_loss / len(train_data))\n    train_accuracy.append(100 * running_correct / len(train_data))\n    \n    print(f'Train Loss: {running_loss / len(train_data):.5f}')\n    print(f'Train Acc: {100 * running_correct / len(train_data):.3f}%')\n    \n    wandb.log({\n        'epoch': e,\n        'training_loss': running_loss / len(train_data), \n        'training_accuracy': 100 * running_correct / len(train_data)\n    }, commit=False)\n    \n    model.eval()\n    running_loss = 0\n    running_correct = 0\n    with torch.no_grad():\n        for image, label in valid_loader:\n            image, label = image.to(config['device']), label.to(config['device'])\n            pred = model(image)\n            loss = loss_fn(pred, label)\n            \n            running_correct += (pred.argmax(dim=1) == label.argmax(dim=1)).float().sum().item()\n            running_loss += loss.item()\n            \n    val_loss.append(running_loss / len(valid_data))\n    val_accuracy.append(100 * running_correct / len(valid_data))\n    \n    print(f'Val Loss: {running_loss / len(valid_data):.5f}')\n    print(f'Val Acc: {100 * running_correct / len(valid_data):.3f}%')\n    \n    wandb.log({\n        'validation_loss': running_loss / len(valid_data), \n        'validation_accuracy': 100 * running_correct / len(valid_data)\n    })\n\n    if running_loss < min_val_loss:\n        print('saving model')\n        torch.save(model.state_dict(), 'model.pt')\n        min_val_loss = running_loss","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:23:28.524473Z","iopub.execute_input":"2024-01-14T15:23:28.525046Z","iopub.status.idle":"2024-01-14T18:37:07.573722Z","shell.execute_reply.started":"2024-01-14T15:23:28.524986Z","shell.execute_reply":"2024-01-14T18:37:07.572397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(2, 1)\n\nax[0].plot(list(range(len(train_loss))), train_loss, list(range(len(val_loss))), val_loss)\nax[0].legend([\"Train\", \"Validation\"])\nax[0].set_title(\"Loss\")\nax[1].axis(ymin=0)\n\nax[1].plot(list(range(len(train_accuracy))), train_accuracy, list(range(len(val_accuracy))), val_accuracy)\nax[1].set_title(\"Accuracy\")\nax[1].axis(ymin=0,ymax=105)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T18:37:07.575328Z","iopub.execute_input":"2024-01-14T18:37:07.575752Z","iopub.status.idle":"2024-01-14T18:37:09.100015Z","shell.execute_reply.started":"2024-01-14T18:37:07.575704Z","shell.execute_reply":"2024-01-14T18:37:09.099002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_state_dict(torch.load('model.pt'))","metadata":{"execution":{"iopub.status.busy":"2024-01-14T18:37:09.101654Z","iopub.execute_input":"2024-01-14T18:37:09.103072Z","iopub.status.idle":"2024-01-14T18:37:09.940886Z","shell.execute_reply.started":"2024-01-14T18:37:09.103011Z","shell.execute_reply":"2024-01-14T18:37:09.939914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader = DataLoader(dataset=test_data, batch_size=1, shuffle=False)\nmodel.eval()\nout_data = []\n\nwith torch.no_grad():\n    for i, (image, label) in enumerate(test_loader):       \n        image = image.to(config['device'])\n        output = model(image)\n        pred_is_dog = F.softmax(output.data, dim=1).cpu()[0][1]\n        out_data.append([i+1, pred_is_dog.item()])\n            \nout_df = pd.DataFrame(out_data, columns=['id', 'label'])\nout_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T18:37:09.942410Z","iopub.execute_input":"2024-01-14T18:37:09.943120Z","iopub.status.idle":"2024-01-14T18:38:49.994319Z","shell.execute_reply.started":"2024-01-14T18:37:09.943082Z","shell.execute_reply":"2024-01-14T18:38:49.993306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_df","metadata":{"execution":{"iopub.status.busy":"2024-01-13T21:44:30.759441Z","iopub.status.idle":"2024-01-13T21:44:30.760387Z","shell.execute_reply.started":"2024-01-13T21:44:30.760096Z","shell.execute_reply":"2024-01-13T21:44:30.760123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}