{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install --upgrade tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-04T07:32:01.657104Z","iopub.execute_input":"2023-11-04T07:32:01.657812Z","iopub.status.idle":"2023-11-04T07:32:14.391686Z","shell.execute_reply.started":"2023-11-04T07:32:01.657777Z","shell.execute_reply":"2023-11-04T07:32:14.390580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%pip -q install vit_pytorch linformer","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:35:57.221463Z","iopub.execute_input":"2023-11-04T07:35:57.222360Z","iopub.status.idle":"2023-11-04T07:36:09.639583Z","shell.execute_reply.started":"2023-11-04T07:35:57.222314Z","shell.execute_reply":"2023-11-04T07:36:09.638484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/Jack-Etheredge/early_convolutions_vit_pytorch.git","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:36:35.549715Z","iopub.execute_input":"2023-11-04T07:36:35.550631Z","iopub.status.idle":"2023-11-04T07:36:37.281886Z","shell.execute_reply.started":"2023-11-04T07:36:35.550591Z","shell.execute_reply":"2023-11-04T07:36:37.280870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%pip install -e /kaggle/working/early_convolutions_vit_pytorch # install the repo as an editable package in case you want to play with the code","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:37:16.077629Z","iopub.execute_input":"2023-11-04T07:37:16.077996Z","iopub.status.idle":"2023-11-04T07:37:30.965489Z","shell.execute_reply.started":"2023-11-04T07:37:16.077967Z","shell.execute_reply":"2023-11-04T07:37:30.964339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import print_function\n\nimport glob\nfrom itertools import chain\nimport os\nimport random\nimport zipfile\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom linformer import Linformer\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom torch.optim.lr_scheduler import StepLR\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms\nfrom tqdm.notebook import tqdm\n\nfrom early_convolutions_vit_pytorch.vitc.early_convolutions import EarlyConvViT","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:38:24.953982Z","iopub.execute_input":"2023-11-04T07:38:24.954740Z","iopub.status.idle":"2023-11-04T07:38:26.653448Z","shell.execute_reply.started":"2023-11-04T07:38:24.954710Z","shell.execute_reply":"2023-11-04T07:38:26.652658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Torch: {torch.__version__}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:38:49.034534Z","iopub.execute_input":"2023-11-04T07:38:49.035424Z","iopub.status.idle":"2023-11-04T07:38:49.040453Z","shell.execute_reply.started":"2023-11-04T07:38:49.035375Z","shell.execute_reply":"2023-11-04T07:38:49.039312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training settings\nbatch_size = 64\nepochs = 60\nlr = 3e-5\ngamma = 0.7\nseed = 42","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:39:19.587153Z","iopub.execute_input":"2023-11-04T07:39:19.587654Z","iopub.status.idle":"2023-11-04T07:39:19.592301Z","shell.execute_reply.started":"2023-11-04T07:39:19.587620Z","shell.execute_reply":"2023-11-04T07:39:19.591305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:39:26.469298Z","iopub.execute_input":"2023-11-04T07:39:26.470136Z","iopub.status.idle":"2023-11-04T07:39:26.480353Z","shell.execute_reply.started":"2023-11-04T07:39:26.470103Z","shell.execute_reply":"2023-11-04T07:39:26.479544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n    device = 'cuda'\nelse:\n    device = 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:39:31.913056Z","iopub.execute_input":"2023-11-04T07:39:31.913922Z","iopub.status.idle":"2023-11-04T07:39:31.945248Z","shell.execute_reply.started":"2023-11-04T07:39:31.913885Z","shell.execute_reply":"2023-11-04T07:39:31.944260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('data', exist_ok=True)\ntrain_dir = 'data/train'\ntest_dir = 'data/test'\n\n# download the data from https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/data to /<repo>/examples/\nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip') as train_zip:\n    train_zip.extractall('data')\n    \nwith zipfile.ZipFile('/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip') as test_zip:\n    test_zip.extractall('data')","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:40:25.146967Z","iopub.execute_input":"2023-11-04T07:40:25.147342Z","iopub.status.idle":"2023-11-04T07:40:43.613363Z","shell.execute_reply.started":"2023-11-04T07:40:25.147303Z","shell.execute_reply":"2023-11-04T07:40:43.612514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:40:43.614832Z","iopub.execute_input":"2023-11-04T07:40:43.615129Z","iopub.status.idle":"2023-11-04T07:40:43.754635Z","shell.execute_reply.started":"2023-11-04T07:40:43.615101Z","shell.execute_reply":"2023-11-04T07:40:43.753768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train Data: {len(train_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:40:46.618808Z","iopub.execute_input":"2023-11-04T07:40:46.619667Z","iopub.status.idle":"2023-11-04T07:40:46.624233Z","shell.execute_reply.started":"2023-11-04T07:40:46.619636Z","shell.execute_reply":"2023-11-04T07:40:46.623229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [path.split('/')[-1].split('.')[0] for path in train_list]","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:40:52.249449Z","iopub.execute_input":"2023-11-04T07:40:52.250338Z","iopub.status.idle":"2023-11-04T07:40:52.270026Z","shell.execute_reply.started":"2023-11-04T07:40:52.250302Z","shell.execute_reply":"2023-11-04T07:40:52.268959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_idx = np.random.randint(1, len(train_list), size=9)\nfig, axes = plt.subplots(3, 3, figsize=(16, 12))\n\nfor idx, ax in enumerate(axes.ravel()):\n    img = Image.open(train_list[idx])\n    ax.set_title(labels[idx])\n    ax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:41:01.942537Z","iopub.execute_input":"2023-11-04T07:41:01.943714Z","iopub.status.idle":"2023-11-04T07:41:04.387227Z","shell.execute_reply.started":"2023-11-04T07:41:01.943678Z","shell.execute_reply":"2023-11-04T07:41:04.386152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_list, valid_list = train_test_split(train_list, \n                                          test_size=0.2,\n                                          stratify=labels,\n                                          random_state=seed)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:41:12.162023Z","iopub.execute_input":"2023-11-04T07:41:12.162398Z","iopub.status.idle":"2023-11-04T07:41:12.194261Z","shell.execute_reply.started":"2023-11-04T07:41:12.162368Z","shell.execute_reply":"2023-11-04T07:41:12.193448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train Data: {len(train_list)}\")\nprint(f\"Validation Data: {len(valid_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:41:18.045809Z","iopub.execute_input":"2023-11-04T07:41:18.046182Z","iopub.status.idle":"2023-11-04T07:41:18.052224Z","shell.execute_reply.started":"2023-11-04T07:41:18.046153Z","shell.execute_reply":"2023-11-04T07:41:18.051181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transforms = transforms.Compose(\n    [\n        transforms.Resize((224, 224)),\n        transforms.RandomResizedCrop(224),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n    ]\n)\n\nval_transforms = transforms.Compose(\n    [\n        transforms.Resize(256),\n        transforms.CenterCrop(224),\n        transforms.ToTensor(),\n    ]\n)\n\n\ntest_transforms = transforms.Compose(\n    [\n        transforms.Resize(256),\n        transforms.CenterCrop(224),\n        transforms.ToTensor(),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:41:27.319973Z","iopub.execute_input":"2023-11-04T07:41:27.320668Z","iopub.status.idle":"2023-11-04T07:41:27.327034Z","shell.execute_reply.started":"2023-11-04T07:41:27.320632Z","shell.execute_reply":"2023-11-04T07:41:27.326012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n\n    def __len__(self):\n        self.filelength = len(self.file_list)\n        return self.filelength\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        img_transformed = self.transform(img)\n\n        label = img_path.split(\"/\")[-1].split(\".\")[0]\n        label = 1 if label == \"dog\" else 0\n\n        return img_transformed, label","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:41:33.686235Z","iopub.execute_input":"2023-11-04T07:41:33.687106Z","iopub.status.idle":"2023-11-04T07:41:33.694805Z","shell.execute_reply.started":"2023-11-04T07:41:33.687072Z","shell.execute_reply":"2023-11-04T07:41:33.693768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = CatsDogsDataset(train_list, transform=train_transforms)\nvalid_data = CatsDogsDataset(valid_list, transform=test_transforms)\ntest_data = CatsDogsDataset(test_list, transform=test_transforms)\n\ntrain_loader = DataLoader(dataset = train_data, batch_size=batch_size, shuffle=True )\nvalid_loader = DataLoader(dataset = valid_data, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset = test_data, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:41:40.820589Z","iopub.execute_input":"2023-11-04T07:41:40.820951Z","iopub.status.idle":"2023-11-04T07:41:40.827062Z","shell.execute_reply.started":"2023-11-04T07:41:40.820919Z","shell.execute_reply":"2023-11-04T07:41:40.826136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_data), len(train_loader))\nprint(len(valid_data), len(valid_loader))","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:41:52.509241Z","iopub.execute_input":"2023-11-04T07:41:52.510105Z","iopub.status.idle":"2023-11-04T07:41:52.514976Z","shell.execute_reply.started":"2023-11-04T07:41:52.510070Z","shell.execute_reply":"2023-11-04T07:41:52.513980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EarlyConvViT(\n    dim=128,\n    num_classes=2,\n    depth = 6,\n    heads = 16,\n    mlp_dim = 2048,\n    channels=3,\n).to(device)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:41:58.733020Z","iopub.execute_input":"2023-11-04T07:41:58.733395Z","iopub.status.idle":"2023-11-04T07:42:01.925797Z","shell.execute_reply.started":"2023-11-04T07:41:58.733364Z","shell.execute_reply":"2023-11-04T07:42:01.924754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loss function\ncriterion = nn.CrossEntropyLoss()\n# optimizer\noptimizer = optim.Adam(model.parameters(), lr=lr)\n# scheduler\nscheduler = StepLR(optimizer, step_size=1, gamma=gamma)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:42:13.886285Z","iopub.execute_input":"2023-11-04T07:42:13.886955Z","iopub.status.idle":"2023-11-04T07:42:13.892846Z","shell.execute_reply.started":"2023-11-04T07:42:13.886919Z","shell.execute_reply":"2023-11-04T07:42:13.891805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(epochs):\n    epoch_loss = 0\n    epoch_accuracy = 0\n\n    for data, label in tqdm(train_loader):\n        data = data.to(device)\n        label = label.to(device)\n\n        output = model(data)\n        loss = criterion(output, label)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        acc = (output.argmax(dim=1) == label).float().mean()\n        epoch_accuracy += acc / len(train_loader)\n        epoch_loss += loss / len(train_loader)\n\n    with torch.no_grad():\n        epoch_val_accuracy = 0\n        epoch_val_loss = 0\n        for data, label in valid_loader:\n            data = data.to(device)\n            label = label.to(device)\n\n            val_output = model(data)\n            val_loss = criterion(val_output, label)\n\n            acc = (val_output.argmax(dim=1) == label).float().mean()\n            epoch_val_accuracy += acc / len(valid_loader)\n            epoch_val_loss += val_loss / len(valid_loader)\n\n    print(\n        f\"Epoch : {epoch+1} - loss : {epoch_loss:.4f} - acc: {epoch_accuracy:.4f} - val_loss : {epoch_val_loss:.4f} - val_acc: {epoch_val_accuracy:.4f}\\n\"\n    )","metadata":{"execution":{"iopub.status.busy":"2023-11-04T07:42:25.177159Z","iopub.execute_input":"2023-11-04T07:42:25.177512Z","iopub.status.idle":"2023-11-04T09:34:39.038959Z","shell.execute_reply.started":"2023-11-04T07:42:25.177483Z","shell.execute_reply":"2023-11-04T09:34:39.037969Z"},"trusted":true},"execution_count":null,"outputs":[]}]}