{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-12-11T15:43:27.643722Z","iopub.execute_input":"2022-12-11T15:43:27.644350Z","iopub.status.idle":"2022-12-11T15:43:27.670662Z","shell.execute_reply.started":"2022-12-11T15:43:27.644269Z","shell.execute_reply":"2022-12-11T15:43:27.670004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:43:27.671835Z","iopub.execute_input":"2022-12-11T15:43:27.672033Z","iopub.status.idle":"2022-12-11T15:43:39.906590Z","shell.execute_reply.started":"2022-12-11T15:43:27.672007Z","shell.execute_reply":"2022-12-11T15:43:39.905763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install wandb --upgrade","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:43:39.908210Z","iopub.execute_input":"2022-12-11T15:43:39.908492Z","iopub.status.idle":"2022-12-11T15:43:53.801785Z","shell.execute_reply.started":"2022-12-11T15:43:39.908460Z","shell.execute_reply":"2022-12-11T15:43:53.800721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nwandb.init(project='Dogs_VS_Cats', save_code=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:44:32.385920Z","iopub.execute_input":"2022-12-11T15:44:32.386867Z","iopub.status.idle":"2022-12-11T15:44:49.195459Z","shell.execute_reply.started":"2022-12-11T15:44:32.386816Z","shell.execute_reply":"2022-12-11T15:44:49.194732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"train_dir = 'train'\ntest_dir = 'test'\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as train_zip:\n    train_zip.extractall('')\n    \nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as test_zip:\n    test_zip.extractall('')\ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\nprint(f\"Train Data: {len(train_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:02.817014Z","iopub.execute_input":"2022-12-11T15:45:02.817335Z","iopub.status.idle":"2022-12-11T15:45:20.124751Z","shell.execute_reply.started":"2022-12-11T15:45:02.817296Z","shell.execute_reply":"2022-12-11T15:45:20.124042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list[1]","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:20.126324Z","iopub.execute_input":"2022-12-11T15:45:20.126861Z","iopub.status.idle":"2022-12-11T15:45:20.505377Z","shell.execute_reply.started":"2022-12-11T15:45:20.126825Z","shell.execute_reply":"2022-12-11T15:45:20.504695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [path.split('/')[-1].split('.')[0] for path in train_list]","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:20.506442Z","iopub.execute_input":"2022-12-11T15:45:20.506662Z","iopub.status.idle":"2022-12-11T15:45:20.879224Z","shell.execute_reply.started":"2022-12-11T15:45:20.506634Z","shell.execute_reply":"2022-12-11T15:45:20.878534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot random image with their label","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1, len(train_list), size=9)\nfig, axes = plt.subplots(3, 3, figsize=(16, 12))\n\nfor idx, ax in enumerate(axes.ravel()):\n    img = Image.open(train_list[idx])\n    ax.set_title(labels[idx])\n    ax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:20.880957Z","iopub.execute_input":"2022-12-11T15:45:20.881789Z","iopub.status.idle":"2022-12-11T15:45:22.966519Z","shell.execute_reply.started":"2022-12-11T15:45:20.881742Z","shell.execute_reply":"2022-12-11T15:45:22.965808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Use Sklearn to split data","metadata":{}},{"cell_type":"code","source":"train_list, valid_list = train_test_split(train_list, \n                                          test_size=0.2,\n                                          stratify=labels,\n                                          random_state=0)\nprint(f\"Train Data: {len(train_list)}\")\nprint(f\"Validation Data: {len(valid_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:22.967704Z","iopub.execute_input":"2022-12-11T15:45:22.967961Z","iopub.status.idle":"2022-12-11T15:45:23.299535Z","shell.execute_reply.started":"2022-12-11T15:45:22.967930Z","shell.execute_reply":"2022-12-11T15:45:23.298945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will discuss this in more detail in a near future...","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n        transforms.Resize(128), # makes it easier for the GPU\n        transforms.RandomResizedCrop(112),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor()])\n\nval_transforms = transforms.Compose([\n        transforms.Resize(128),\n        transforms.CenterCrop(112),\n        transforms.ToTensor()])\n\n\ntest_transforms = transforms.Compose([\n        transforms.Resize(128),\n        transforms.CenterCrop(112),\n        transforms.ToTensor()])","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:23.300554Z","iopub.execute_input":"2022-12-11T15:45:23.300765Z","iopub.status.idle":"2022-12-11T15:45:23.836974Z","shell.execute_reply.started":"2022-12-11T15:45:23.300739Z","shell.execute_reply":"2022-12-11T15:45:23.836194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the dataset using PIL to read image","metadata":{}},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, file_list, transform=None, use_gpu=torch.cuda.is_available(), test=False):\n        self.file_list = file_list\n        self.transform = transform\n        self.filelength = len(file_list)\n        self.use_gpu = use_gpu\n        self.test = test\n\n    def __len__(self):\n        return self.filelength\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        img_transformed = self.transform(img)\n        label = img_path.split(\"/\")[-1].split(\".\")[0]\n        if not self.test:\n            label = 1 if label == \"dog\" else 0\n        else:\n            label = int(label)\n        label = torch.tensor(label)\n        if self.use_gpu:\n            img_transformed = img_transformed.cuda()\n            label = label.cuda()\n        return img_transformed, label","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:23.838141Z","iopub.execute_input":"2022-12-11T15:45:23.838391Z","iopub.status.idle":"2022-12-11T15:45:24.252993Z","shell.execute_reply.started":"2022-12-11T15:45:23.838363Z","shell.execute_reply":"2022-12-11T15:45:24.252278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = CatsDogsDataset(train_list, transform=train_transforms)\nvalid_data = CatsDogsDataset(valid_list, transform=val_transforms)\ntest_data = CatsDogsDataset(test_list, transform=test_transforms, test=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:24.254086Z","iopub.execute_input":"2022-12-11T15:45:24.254305Z","iopub.status.idle":"2022-12-11T15:45:24.889542Z","shell.execute_reply.started":"2022-12-11T15:45:24.254277Z","shell.execute_reply":"2022-12-11T15:45:24.888848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create dataloader, you can modify the batch size if needed","metadata":{}},{"cell_type":"code","source":"batch_size = 32\ntrain_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(dataset=valid_data, batch_size=batch_size, shuffle=False)\ntest_loader = DataLoader(dataset=test_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T15:45:24.890583Z","iopub.execute_input":"2022-12-11T15:45:24.890787Z","iopub.status.idle":"2022-12-11T15:45:25.270621Z","shell.execute_reply.started":"2022-12-11T15:45:24.890761Z","shell.execute_reply":"2022-12-11T15:45:25.269932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TODO: As usual, train your model","metadata":{}},{"cell_type":"markdown","source":"## Model Training","metadata":{}},{"cell_type":"markdown","source":"### 1. AlexNet","metadata":{}},{"cell_type":"code","source":"def init_layer_xavier(m):\n    if isinstance(m, nn.Linear):\n        torch.nn.init.xavier_normal_(m.weight, nn.init.calculate_gain('relu'))\n        torch.nn.init.constant_(m.bias, 0)\n\nclass AlexNet(nn.Module):\n    \n    def __init__(self):\n        super(AlexNet, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels= 96, kernel_size= 11, stride=4)\n        self.maxpool1 = nn.MaxPool2d(kernel_size=3, stride=2)\n        self.conv2 = nn.Conv2d(in_channels=96, out_channels=256, kernel_size=5, stride= 1, padding= 2)\n        self.maxpool2 = nn.MaxPool2d(kernel_size=3, stride=2)\n        self.conv3 = nn.Conv2d(in_channels=256, out_channels=384, kernel_size=3, stride= 1, padding= 1)\n        self.conv4 = nn.Conv2d(in_channels=384, out_channels=384, kernel_size=3, stride=1, padding=1)\n        self.conv5 = nn.Conv2d(in_channels=384, out_channels=256, kernel_size=3, stride=1, padding=1)\n        self.maxpool3 = nn.MaxPool2d(kernel_size=3, stride=2)\n        self.fc1  = nn.Linear(in_features= 1024, out_features= 4096)\n        self.fc2  = nn.Linear(in_features= 4096, out_features= 4096)\n        self.fc3 = nn.Linear(in_features=4096 , out_features=1)\n    \n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n        x = self.maxpool1(x)\n        x = F.relu(self.conv2(x))\n        x = self.maxpool2(x)\n        x = F.relu(self.conv3(x))\n        x = F.relu(self.conv4(x))\n        x = F.relu(self.conv5(x))\n        x = self.maxpool3(x)\n        x = x.reshape(x.shape[0], -1)\n        x = F.relu(self.fc1(x))\n        x = F.dropout(x, training=self.training)\n        x = F.relu(self.fc2(x))\n        return F.sigmoid(x)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T16:41:21.681579Z","iopub.execute_input":"2022-12-11T16:41:21.682441Z","iopub.status.idle":"2022-12-11T16:41:22.094125Z","shell.execute_reply.started":"2022-12-11T16:41:21.682398Z","shell.execute_reply":"2022-12-11T16:41:22.093251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 36 # should be more than enought, but can be changed\nlr = 3e-3 # e.q to 0.003, you can change it if needed\nwd = 1e-4 # Weight Decay l2\ncriterion = nn.CrossEntropyLoss()\n\noptimizer = optim.AdamW(net.parameters(), lr=lr, weight_decay=wd)\n\nnet = AlexNet()\nnet.apply(init_layer_xavier)\n\nfor epoch in range(num_epochs):\n    training_loss = 0\n    net.train()\n    for X, y in train_loader:\n        optimizer.zero_grad() # please don't forget!\n\n        y_hat = net(X)   # Forward pass\n        loss = criterion(y_hat, y)\n        loss.backward()\n        optimizer.step()\n\n        training_loss += loss.item()\n\n        score, predicted = torch.max(y_hat.data, 1)\n        acc_train = (y == predicted).sum().float() / len(y)\n        wandb.log({\n            'alexnet/loss_train': loss.item(),\n            'alexnet/accuracy_train': acc_train,\n        })\n\n    with torch.no_grad():\n        validation_loss = 0\n        net.eval()   # For validation loop\n        for Z, j in valid_loader:\n            j_hat = net(Z)\n            loss_val = criterion(j_hat, j)\n\n            validation_loss += loss_val.item()\n\n            score, predicted = torch.max(j_hat, 1)\n            acc_val = (j == predicted).sum().float() / len(j)\n            wandb.log({\n                'alexnet/loss_val': loss_val.item(),\n                'alexnet/accuracy_val': acc_val,\n            })\n\n# SAVE THE MODEL\ntorch.save(net.state_dict(), 'mutabay_alexnet_dogsvscats_model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-12-11T17:22:27.053089Z","iopub.execute_input":"2022-12-11T17:22:27.054056Z","iopub.status.idle":"2022-12-11T19:15:33.814335Z","shell.execute_reply.started":"2022-12-11T17:22:27.054016Z","shell.execute_reply":"2022-12-11T19:15:33.813556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loop to generate the submission file","metadata":{}},{"cell_type":"code","source":"prob = []\nnet.eval()\nwith torch.no_grad():\n    for X, y in test_loader:\n        y_pred = net(X).flatten().tolist()\n        #y_pred = F.softmax(model(X), dim=1)[:, 1].tolist()\n        prob = prob + y_pred \n\nidx = [i for i in range(1,len(prob)+1)]   \nsubmission = pd.DataFrame({'id':idx,'label':prob})\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T19:52:36.777721Z","iopub.execute_input":"2022-12-11T19:52:36.778024Z","iopub.status.idle":"2022-12-11T19:58:45.614678Z","shell.execute_reply.started":"2022-12-11T19:52:36.777994Z","shell.execute_reply":"2022-12-11T19:58:45.613923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission_new.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T20:00:24.506653Z","iopub.execute_input":"2022-12-11T20:00:24.507617Z","iopub.status.idle":"2022-12-11T20:01:27.500288Z","shell.execute_reply.started":"2022-12-11T20:00:24.507575Z","shell.execute_reply":"2022-12-11T20:01:27.499166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    net.eval()\n    test_pred = torch.LongTensor()\n    result_dict = []\n    for i, data in enumerate(test_loader):\n        x, y = data\n        output = net(x)\n        predicted = output.cpu()\n        predicted = torch.where(predicted > 0.5, torch.ones_like(predicted), torch.zeros_like(predicted))\n        for i in range(len(y)):\n            result_dict.append({'id': y[i].cpu().item(), 'label': predicted[i].item()})\n    out_df = pd.DataFrame(np.c_[np.arange(1, len(test_list)+1)[:, None],\n                                test_pred.cpu().numpy()], columns=['ImageId', 'Label'])\n    out_df.to_csv('submission.csv', index=False)\n    print(out_df)","metadata":{"execution":{"iopub.status.busy":"2022-12-11T19:50:24.243751Z","iopub.execute_input":"2022-12-11T19:50:24.244822Z","iopub.status.idle":"2022-12-11T19:50:24.851109Z","shell.execute_reply.started":"2022-12-11T19:50:24.244767Z","shell.execute_reply":"2022-12-11T19:50:24.850011Z"},"trusted":true},"execution_count":null,"outputs":[]}]}