{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/jerzypawlik/cats-dogs-redux\nhttps://wandb.ai/jerze/lab_cats&dogs?workspace=user-jerzyjerzu\ndone in collaboration of Vid Rebol and Jerzy Pawlik","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-13T11:10:47.382734Z","iopub.execute_input":"2024-01-13T11:10:47.383472Z","iopub.status.idle":"2024-01-13T11:10:47.774073Z","shell.execute_reply.started":"2024-01-13T11:10:47.383433Z","shell.execute_reply":"2024-01-13T11:10:47.773092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:10:47.776124Z","iopub.execute_input":"2024-01-13T11:10:47.776982Z","iopub.status.idle":"2024-01-13T11:10:50.057151Z","shell.execute_reply.started":"2024-01-13T11:10:47.776941Z","shell.execute_reply":"2024-01-13T11:10:50.056217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO: wnadb\nimport wandb\nwandb.init(project='lab_cats&dogs', save_code=True,mode='disabled')\n#wandb.init(project='_MLP', save_code=True, mode ='disabled')","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:10:50.058327Z","iopub.execute_input":"2024-01-13T11:10:50.058774Z","iopub.status.idle":"2024-01-13T11:11:24.786014Z","shell.execute_reply.started":"2024-01-13T11:10:50.058745Z","shell.execute_reply":"2024-01-13T11:11:24.784954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"train_dir = 'train'\ntest_dir = 'test'\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as train_zip:\n    train_zip.extractall('')\n    \nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as test_zip:\n    test_zip.extractall('')\ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\nprint(f\"Train Data: {len(train_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:24.789030Z","iopub.execute_input":"2024-01-13T11:11:24.789423Z","iopub.status.idle":"2024-01-13T11:11:36.236231Z","shell.execute_reply.started":"2024-01-13T11:11:24.789387Z","shell.execute_reply":"2024-01-13T11:11:36.235235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [path.split('/')[-1].split('.')[0] for path in train_list]","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:36.237491Z","iopub.execute_input":"2024-01-13T11:11:36.237784Z","iopub.status.idle":"2024-01-13T11:11:36.970049Z","shell.execute_reply.started":"2024-01-13T11:11:36.237756Z","shell.execute_reply":"2024-01-13T11:11:36.969064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot random image with their label","metadata":{}},{"cell_type":"code","source":"train_list, valid_list = train_test_split(train_list, \n                                          test_size=0.2,\n                                          stratify=labels,\n                                          random_state=0)","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:36.971507Z","iopub.execute_input":"2024-01-13T11:11:36.972205Z","iopub.status.idle":"2024-01-13T11:11:37.563603Z","shell.execute_reply.started":"2024-01-13T11:11:36.972175Z","shell.execute_reply":"2024-01-13T11:11:37.562642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_idx = np.random.randint(1, len(train_list), size=9)\nfig, axes = plt.subplots(3, 3, figsize=(16, 12))\n\nfor idx, ax in enumerate(axes.ravel()):\n    img = Image.open(train_list[idx])\n    ax.set_title(labels[idx])\n    ax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:37.564814Z","iopub.execute_input":"2024-01-13T11:11:37.565212Z","iopub.status.idle":"2024-01-13T11:11:40.692195Z","shell.execute_reply.started":"2024-01-13T11:11:37.565173Z","shell.execute_reply":"2024-01-13T11:11:40.691210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Use Sklearn to split the training data","metadata":{}},{"cell_type":"code","source":"print(f\"Train Data: {len(train_list)}\")\nprint(f\"Validation Data: {len(valid_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:40.693445Z","iopub.execute_input":"2024-01-13T11:11:40.693807Z","iopub.status.idle":"2024-01-13T11:11:41.322799Z","shell.execute_reply.started":"2024-01-13T11:11:40.693769Z","shell.execute_reply":"2024-01-13T11:11:41.321674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list[:3]","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:41.324277Z","iopub.execute_input":"2024-01-13T11:11:41.324640Z","iopub.status.idle":"2024-01-13T11:11:41.906313Z","shell.execute_reply.started":"2024-01-13T11:11:41.324603Z","shell.execute_reply":"2024-01-13T11:11:41.905312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will discuss this in more detail in a near future...","metadata":{}},{"cell_type":"code","source":"# TODO add augmentations\ntrain_transforms = transforms.Compose([\n    #transforms.Resize((256,256)),\n    #for alexNet\n    transforms.Resize((224,224)),\n     transforms.ToTensor(),\n     # Normalize(mean, STD) for every channel of the image \n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n])\n\ntest_transforms = transforms.Compose([\n     #transforms.Resize((256,256)),\n    #for alexNet\n     transforms.Resize((224,224)),\n     transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n])","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:41.909402Z","iopub.execute_input":"2024-01-13T11:11:41.909719Z","iopub.status.idle":"2024-01-13T11:11:42.541418Z","shell.execute_reply.started":"2024-01-13T11:11:41.909675Z","shell.execute_reply":"2024-01-13T11:11:42.540385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the dataset using PIL to read image","metadata":{}},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n        self.filelength = len(file_list)\n\n    def __len__(self):\n        return self.filelength\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        img_transformed = self.transform(img)\n        label = img_path.split(\"/\")[-1].split(\".\")[0]\n        label = 1 if label == \"dog\" else 0\n        return img_transformed, label","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:42.542800Z","iopub.execute_input":"2024-01-13T11:11:42.543215Z","iopub.status.idle":"2024-01-13T11:11:43.125135Z","shell.execute_reply.started":"2024-01-13T11:11:42.543179Z","shell.execute_reply":"2024-01-13T11:11:43.123918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = CatsDogsDataset(train_list, transform=train_transforms)\nvalid_data = CatsDogsDataset(valid_list, transform=test_transforms)\ntest_data = CatsDogsDataset(test_list, transform=test_transforms)","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:43.126661Z","iopub.execute_input":"2024-01-13T11:11:43.127039Z","iopub.status.idle":"2024-01-13T11:11:43.755598Z","shell.execute_reply.started":"2024-01-13T11:11:43.127002Z","shell.execute_reply":"2024-01-13T11:11:43.754381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create dataloader, you can modify the batch size if needed","metadata":{}},{"cell_type":"code","source":"batch_size = 32\ntrain_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(dataset=valid_data, batch_size=batch_size, shuffle=False)\ntest_loader = DataLoader(dataset=test_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:43.760033Z","iopub.execute_input":"2024-01-13T11:11:43.760455Z","iopub.status.idle":"2024-01-13T11:11:44.415290Z","shell.execute_reply.started":"2024-01-13T11:11:43.760410Z","shell.execute_reply":"2024-01-13T11:11:44.414205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Improved version of the network\n\nclass CNN(nn.Module):\n     def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(3, 9, 5)\n        self.pool1 = nn.MaxPool2d(7, 4)\n        self.pool2 = nn.MaxPool2d(2, 2)\n        self.conv2 = nn.Conv2d(9, 18, 5)\n        self.fc1 = nn.Linear(3042, 3042)\n        self.dropout = nn.Dropout(config.dropout)\n        self.fc2 = nn.Linear(3042, 512)\n        self.fc3 = nn.Linear(512, 1)\n\n    def forward(self, x):\n        x = self.pool1(F.relu(self.conv1(x)))\n        x = self.pool2(F.relu(self.conv2(x)))\n        x = torch.flatten(x, 1) # flatten all dimensions except batch\n        x = F.relu(self.fc1(x))\n        x = self.dropout(x)\n        x = F.relu(self.fc2(x))\n        x = self.dropout(x)\n        x = F.sigmoid(self.fc3(x))\n        return x\n\ndef _weights_init(m):\n    if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):\n        torch.nn.init.xavier_normal_(m.weight)\n        m.bias.data.zero_()\n ","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:44.416511Z","iopub.execute_input":"2024-01-13T11:11:44.416792Z","iopub.status.idle":"2024-01-13T11:11:45.008564Z","shell.execute_reply.started":"2024-01-13T11:11:44.416765Z","shell.execute_reply":"2024-01-13T11:11:45.007807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AlexNet(nn.Module):\n    \n    def __init__(self):\n        super(AlexNet, self).__init__()\n        self.conv1 = nn.Conv2d(3, 96, 11, stride=4)\n        self.maxPool = nn.MaxPool2d(3, stride=2)\n        self.conv2 = nn.Conv2d(96, 256, 5, padding=2)\n        self.conv3 = nn.Conv2d(256, 384, 3, padding=1)\n        self.conv4 = nn.Conv2d(384, 384, 3, padding=1)\n        self.conv5 = nn.Conv2d(384, 256, 3, padding=1)\n        self.fc1 = nn.Linear(5 * 5 * 256, 4096)\n        self.fc2 = nn.Linear(4096, 1000)\n        self.fc3 = nn.Linear(1000, 1)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(p=0.5)\n    \n    def forward(self, x):\n        x = self.maxPool(self.relu(self.conv1(x)))\n        x = self.maxPool(self.relu(self.conv2(x)))\n        x = self.relu(self.conv3(x))\n        x = self.relu(self.conv4(x))\n        x = self.relu(self.conv5(x))\n        x = self.maxPool(x)\n        x = x.reshape(x.size(0), -1)\n        x = self.dropout(self.relu(self.fc1(x)))\n        x = self.dropout(self.relu(self.fc2(x)))\n        return F.sigmoid(self.fc3(x))","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:45.009702Z","iopub.execute_input":"2024-01-13T11:11:45.010000Z","iopub.status.idle":"2024-01-13T11:11:45.612249Z","shell.execute_reply.started":"2024-01-13T11:11:45.009973Z","shell.execute_reply":"2024-01-13T11:11:45.611143Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# not tried yet\ndef my_accuracy(predictions, labels):\n    predictions1 = torch.abs(torch.round(predictions))\n    #print(labels)\n    correct = (predictions1 == labels)\n    #print(correct)\n    acc = sum(correct) / len(predictions1)\n    return acc.item()","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:45.613755Z","iopub.execute_input":"2024-01-13T11:11:45.614219Z","iopub.status.idle":"2024-01-13T11:11:46.270530Z","shell.execute_reply.started":"2024-01-13T11:11:45.614174Z","shell.execute_reply":"2024-01-13T11:11:46.269464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(torch.cuda.is_available())\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:46.271723Z","iopub.execute_input":"2024-01-13T11:11:46.272046Z","iopub.status.idle":"2024-01-13T11:11:46.900316Z","shell.execute_reply.started":"2024-01-13T11:11:46.272012Z","shell.execute_reply":"2024-01-13T11:11:46.899240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn = AlexNet()\ncnn.apply(_weights_init)\ncnn.to(device)\n\nd = next(cnn.parameters()).device\n\nif d.type == 'cuda':\n    print(\"Model is on GPU\")\nelse:\n    print(\"Model is on CPU\")","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:46.901521Z","iopub.execute_input":"2024-01-13T11:11:46.901819Z","iopub.status.idle":"2024-01-13T11:11:48.133703Z","shell.execute_reply.started":"2024-01-13T11:11:46.901787Z","shell.execute_reply":"2024-01-13T11:11:48.132572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 20 # should be more than enough, but can be changed\nlr = 7e-3 # e.q to 0.007, you can change it if needed\n\n# to be adjusted\ncriterion = nn.BCELoss()\noptimizer = torch.optim.SGD(cnn.parameters(), lr=lr)\n\nfor epoch in range(num_epochs):\n    training_loss_epoch = 0\n    training_accuracy_epoch = 0\n    cnt  = 0\n    for train_features_batch, train_labels_batch in train_loader:\n        train_features_batch, train_labels_batch = train_features_batch.to(device), train_labels_batch.to(device)\n        #uncomment to put it on gpu\n        #train_features_batch, train_labels_batch = train_features_batch.to(device),train_labels_batch.to(device)\n        \n        optimizer.zero_grad() # please don't forget!\n        #?\n        #reshaped_train_features_batch = train_features_batch.reshape(len(train_features_batch), -1)\n        \n        y_hat = cnn(train_features_batch)\n        train_labels_batch = train_labels_batch.reshape(len(train_labels_batch),1)\n        train_labels_batch = train_labels_batch.float()\n        \n        \n        training_loss = criterion(y_hat, train_labels_batch)\n        training_loss.backward() # remember: You need to tell w.r.t what the gradient is computed\n        optimizer.step()\n        \n        training_loss_epoch += training_loss.item()\n        \n        #print(training_loss)\n        #wandb.log({'training_loss_batch': training_loss.item()})\n        \n        train_accuracy = my_accuracy(y_hat, train_labels_batch)\n        training_accuracy_epoch += train_accuracy\n        #print(\"train iteration\" +str(cnt))\n        cnt+=1\n        \n    validation_loss_epoch = 0\n    validation_accuracy_epoch = 0\n    cnt = 0\n    for val_features_batch, val_labels_batch in valid_loader:\n        val_features_batch, val_labels_batch = val_features_batch.to(device), val_labels_batch.to(device)\n        \n        with torch.no_grad():\n            #gpu\n            #val_features_batch, val_labels_batch = val_features_batch.to(device),val_labels_batch.to(device)               \n            \n            #reshaped_val_features_batch = val_features_batch.reshape(len(val_features_batch), -1)\n        \n            y_hat_val = cnn(val_features_batch)\n            val_labels_batch = val_labels_batch.reshape(len(val_labels_batch),1)\n            val_labels_batch = val_labels_batch.float()\n            \n            validation_loss = criterion(y_hat_val, val_labels_batch)\n            validation_loss_epoch += validation_loss.item()\n            \n            val_accuracy = my_accuracy(y_hat_val, val_labels_batch)\n            validation_accuracy_epoch += val_accuracy\n            #wandb.log({'validation_loss_batch': validation_loss.item()})\n            #print(\"val iteration\" +str(cnt))\n            cnt+=1\n    \n    wandb.log({'training_loss_epoch': training_loss_epoch/len(train_loader), 'validation_loss_epoch': validation_loss_epoch/len(valid_loader)})\n    wandb.log({'training_accuracy_epoch': training_accuracy_epoch/len(train_loader), 'validation_accuracy_epoch': validation_accuracy_epoch/len(valid_loader)})\n    print({'training_loss_epoch': training_loss_epoch/len(train_loader), 'validation_loss_epoch': validation_loss_epoch/len(valid_loader)})\n    print({'training_accuracy_epoch': training_accuracy_epoch/len(train_loader), 'validation_accuracy_epoch': validation_accuracy_epoch/len(valid_loader)})\n    if training_loss_epoch/len(train_loader) - validation_loss_epoch/len(valid_loader) < -0.08:\n        break","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:11:48.135294Z","iopub.execute_input":"2024-01-13T11:11:48.136051Z","iopub.status.idle":"2024-01-13T11:39:17.542475Z","shell.execute_reply.started":"2024-01-13T11:11:48.136019Z","shell.execute_reply":"2024-01-13T11:39:17.541489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    cnn.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_loader):\n        test_batch = data[0].to(device)\n        output = cnn(test_batch)\n        #, predicted = torch.max(output.data, 1)\n        predicted = output\n        predicted = predicted.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n    out_df = pd.DataFrame(np.c_[np.arange(1, len(test_list)+1)[:,None], test_pred.numpy()], columns=['id', 'label'])\n    out_df['id'] = out_df['id'].astype(int)\n    out_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-13T11:39:17.543695Z","iopub.execute_input":"2024-01-13T11:39:17.544008Z","iopub.status.idle":"2024-01-13T11:40:02.712114Z","shell.execute_reply.started":"2024-01-13T11:39:17.543982Z","shell.execute_reply":"2024-01-13T11:40:02.711134Z"},"trusted":true},"execution_count":null,"outputs":[]}]}