{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":22881,"databundleVersionId":1552852,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install facenet-pytorch","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:29:38.27355Z","iopub.execute_input":"2024-05-18T08:29:38.273893Z","iopub.status.idle":"2024-05-18T08:32:00.246575Z","shell.execute_reply.started":"2024-05-18T08:29:38.273865Z","shell.execute_reply":"2024-05-18T08:32:00.245642Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting facenet-pytorch\n  Downloading facenet_pytorch-2.6.0-py3-none-any.whl.metadata (12 kB)\nRequirement already satisfied: numpy<2.0.0,>=1.24.0 in /opt/conda/lib/python3.10/site-packages (from facenet-pytorch) (1.26.4)\nCollecting Pillow<10.3.0,>=10.2.0 (from facenet-pytorch)\n  Downloading pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (9.7 kB)\nRequirement already satisfied: requests<3.0.0,>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from facenet-pytorch) (2.31.0)\nCollecting torch<2.3.0,>=2.2.0 (from facenet-pytorch)\n  Downloading torch-2.2.2-cp310-cp310-manylinux1_x86_64.whl.metadata (26 kB)\nCollecting torchvision<0.18.0,>=0.17.0 (from facenet-pytorch)\n  Downloading torchvision-0.17.2-cp310-cp310-manylinux1_x86_64.whl.metadata (6.6 kB)\nRequirement already satisfied: tqdm<5.0.0,>=4.0.0 in /opt/conda/lib/python3.10/site-packages (from facenet-pytorch) (4.66.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.0.0->facenet-pytorch) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.0.0->facenet-pytorch) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.0.0->facenet-pytorch) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3.0.0,>=2.0.0->facenet-pytorch) (2024.2.2)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch<2.3.0,>=2.2.0->facenet-pytorch) (3.13.1)\nRequirement already satisfied: typing-extensions>=4.8.0 in /opt/conda/lib/python3.10/site-packages (from torch<2.3.0,>=2.2.0->facenet-pytorch) (4.9.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch<2.3.0,>=2.2.0->facenet-pytorch) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch<2.3.0,>=2.2.0->facenet-pytorch) (3.2.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch<2.3.0,>=2.2.0->facenet-pytorch) (3.1.2)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch<2.3.0,>=2.2.0->facenet-pytorch) (2024.2.0)\nCollecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cublas-cu12==12.1.3.1 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cufft-cu12==11.0.2.54 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-curand-cu12==10.3.2.106 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\nCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\nCollecting nvidia-nccl-cu12==2.19.3 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\nCollecting nvidia-nvtx-cu12==12.1.105 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\nCollecting triton==2.2.0 (from torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading triton-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.4 kB)\nCollecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch<2.3.0,>=2.2.0->facenet-pytorch)\n  Downloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch<2.3.0,>=2.2.0->facenet-pytorch) (2.1.3)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch<2.3.0,>=2.2.0->facenet-pytorch) (1.3.0)\nDownloading facenet_pytorch-2.6.0-py3-none-any.whl (1.9 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m20.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl (4.5 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m72.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n\u001b[?25hDownloading torch-2.2.2-cp310-cp310-manylinux1_x86_64.whl (755.5 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m755.5/755.5 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m79.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m64.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m38.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m0:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hDownloading triton-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (167.9 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m167.9/167.9 MB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading torchvision-0.17.2-cp310-cp310-manylinux1_x86_64.whl (6.9 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.9/6.9 MB\u001b[0m \u001b[31m80.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m65.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hInstalling collected packages: triton, Pillow, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch, torchvision, facenet-pytorch\n  Attempting uninstall: Pillow\n    Found existing installation: Pillow 9.5.0\n    Uninstalling Pillow-9.5.0:\n      Successfully uninstalled Pillow-9.5.0\n  Attempting uninstall: torch\n    Found existing installation: torch 2.1.2\n    Uninstalling torch-2.1.2:\n      Successfully uninstalled torch-2.1.2\n  Attempting uninstall: torchvision\n    Found existing installation: torchvision 0.16.2\n    Uninstalling torchvision-0.16.2:\n      Successfully uninstalled torchvision-0.16.2\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\nspopt 0.6.0 requires shapely>=2.0.1, but you have shapely 1.8.5.post1 which is incompatible.\nydata-profiling 4.6.4 requires numpy<1.26,>=1.16.0, but you have numpy 1.26.4 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed Pillow-10.2.0 facenet-pytorch-2.6.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 torch-2.2.2 torchvision-0.17.2 triton-2.2.0\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom tqdm import tqdm\nfrom facenet_pytorch import InceptionResnetV1, MTCNN\nfrom torch.utils import data\nfrom torchvision.datasets import ImageFolder\nfrom torch.utils.data import DataLoader\nimport random\n\nfrom torchvision.transforms import v2 \nimport torchvision.transforms as transforms\nfrom torchvision.io import read_image\n\nfrom os import path\nimport os\n\nprint(\"CUDA Available\", torch.cuda.is_available())\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:36:16.546855Z","iopub.execute_input":"2024-05-18T08:36:16.547585Z","iopub.status.idle":"2024-05-18T08:36:16.554745Z","shell.execute_reply.started":"2024-05-18T08:36:16.547552Z","shell.execute_reply":"2024-05-18T08:36:16.553764Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"CUDA Available True\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Loading DataSets","metadata":{}},{"cell_type":"code","source":"BASE_FOLDER = \"/kaggle/input/11-785-fall-20-homework-2-part-2/\"","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:32:05.305726Z","iopub.execute_input":"2024-05-18T08:32:05.306211Z","iopub.status.idle":"2024-05-18T08:32:05.310577Z","shell.execute_reply.started":"2024-05-18T08:32:05.306182Z","shell.execute_reply":"2024-05-18T08:32:05.30966Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":"### Cutting Face out of Image before training","metadata":{}},{"cell_type":"code","source":"mtcnn = MTCNN(select_largest=True, device=device)\nmtcnn.to(device)\nmtcnn = nn.DataParallel(mtcnn)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:32:05.31415Z","iopub.execute_input":"2024-05-18T08:32:05.314431Z","iopub.status.idle":"2024-05-18T08:32:05.484752Z","shell.execute_reply.started":"2024-05-18T08:32:05.314408Z","shell.execute_reply":"2024-05-18T08:32:05.483972Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"class FaceCropTransform:\n    def __init__(self, mtcnn):\n        self.mtcnn = mtcnn\n\n    def __call__(self, img):\n        # Detect face and return the cropped image\n        boxes, _ = self.mtcnn.detect(img)\n        if boxes is None:\n            return None\n        box = boxes[0]\n        img = img.crop(box)\n        return img","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:32:05.485779Z","iopub.execute_input":"2024-05-18T08:32:05.486058Z","iopub.status.idle":"2024-05-18T08:32:05.491412Z","shell.execute_reply.started":"2024-05-18T08:32:05.486034Z","shell.execute_reply":"2024-05-18T08:32:05.490543Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"INPUT_SIZE = (299, 299)\n\ndata_transforms = {\n    \"train\": v2.Compose([\n#         FaceCropTransform(mtcnn),\n        v2.Resize(size=INPUT_SIZE),\n        v2.RandomHorizontalFlip(p=0.5),\n        v2.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.2, hue=0.1),\n        v2.RandomAffine(degrees=0, translate=(0.1, 0.1), scale=None, shear=10),\n        transforms.ToTensor(),\n        v2.ToDtype(torch.float32, scale=True),\n        v2.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ]),\n    \"val\": v2.Compose([\n#         FaceCropTransform(mtcnn),\n        v2.Resize(size=INPUT_SIZE),\n        transforms.ToTensor(),\n        v2.ToDtype(torch.float32, scale=True),\n        v2.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ]),\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:36:19.492263Z","iopub.execute_input":"2024-05-18T08:36:19.493089Z","iopub.status.idle":"2024-05-18T08:36:19.501023Z","shell.execute_reply.started":"2024-05-18T08:36:19.493059Z","shell.execute_reply":"2024-05-18T08:36:19.500058Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"markdown","source":"### Creating DataLoader for Classifier","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:32:05.50441Z","iopub.execute_input":"2024-05-18T08:32:05.505023Z","iopub.status.idle":"2024-05-18T08:32:05.513509Z","shell.execute_reply.started":"2024-05-18T08:32:05.504993Z","shell.execute_reply":"2024-05-18T08:32:05.512809Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"val_ds = ImageFolder(root=path.join(BASE_FOLDER, \"classification_data/val_data\"), transform=data_transforms[\"val\"])\ntest_ds = ImageFolder(root=path.join(BASE_FOLDER, \"classification_data/test_data\"), transform=data_transforms[\"val\"])\ntrain_ds = ImageFolder(root=path.join(BASE_FOLDER, \"classification_data/train_data\"), transform=data_transforms[\"train\"])\nprint(len(train_ds.classes))\n\nval_dl = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=True) \ntest_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=True) \ntrain_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True) \n\nassert len(train_ds.classes) == len(val_ds.classes) == len(test_ds.classes)\nnum_classes = len(train_ds.classes)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:36:24.154666Z","iopub.execute_input":"2024-05-18T08:36:24.155486Z","iopub.status.idle":"2024-05-18T08:36:36.155059Z","shell.execute_reply.started":"2024-05-18T08:36:24.155446Z","shell.execute_reply":"2024-05-18T08:36:36.154073Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stdout","text":"4000\n","output_type":"stream"}]},{"cell_type":"code","source":"X = None\ny = None\nfor data in val_dl:\n    global X\n    global y\n    X = data[0]\n    y = data[1]\n    break","metadata":{"execution":{"iopub.status.busy":"2024-05-18T08:36:21.854475Z","iopub.execute_input":"2024-05-18T08:36:21.855119Z","iopub.status.idle":"2024-05-18T08:36:21.982733Z","shell.execute_reply.started":"2024-05-18T08:36:21.855081Z","shell.execute_reply":"2024-05-18T08:36:21.98142Z"},"trusted":true},"execution_count":14,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mUnidentifiedImageError\u001b[0m                    Traceback (most recent call last)","Cell \u001b[0;32mIn[14], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m      2\u001b[0m y \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m data \u001b[38;5;129;01min\u001b[39;00m val_dl:\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;28;01mglobal\u001b[39;00m X\n\u001b[1;32m      5\u001b[0m     \u001b[38;5;28;01mglobal\u001b[39;00m y\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:631\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    628\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    629\u001b[0m     \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m    630\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset()  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 631\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    632\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m    633\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    634\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    635\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called:\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:675\u001b[0m, in \u001b[0;36m_SingleProcessDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    673\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_next_data\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    674\u001b[0m     index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_next_index()  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[0;32m--> 675\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset_fetcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[1;32m    676\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[1;32m    677\u001b[0m         data \u001b[38;5;241m=\u001b[39m _utils\u001b[38;5;241m.\u001b[39mpin_memory\u001b[38;5;241m.\u001b[39mpin_memory(data, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory_device)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py:51\u001b[0m, in \u001b[0;36m_MapDatasetFetcher.fetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__(possibly_batched_index)\n\u001b[1;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[idx] \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py:51\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__(possibly_batched_index)\n\u001b[1;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torchvision/datasets/folder.py:229\u001b[0m, in \u001b[0;36mDatasetFolder.__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    221\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    222\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[1;32m    223\u001b[0m \u001b[38;5;124;03m    index (int): Index\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[38;5;124;03m    tuple: (sample, target) where target is class_index of the target class.\u001b[39;00m\n\u001b[1;32m    227\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    228\u001b[0m path, target \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msamples[index]\n\u001b[0;32m--> 229\u001b[0m sample \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    230\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtransform \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    231\u001b[0m     sample \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtransform(sample)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torchvision/datasets/folder.py:268\u001b[0m, in \u001b[0;36mdefault_loader\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m    266\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m accimage_loader(path)\n\u001b[1;32m    267\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 268\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mpil_loader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/torchvision/datasets/folder.py:247\u001b[0m, in \u001b[0;36mpil_loader\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m    244\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpil_loader\u001b[39m(path: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Image\u001b[38;5;241m.\u001b[39mImage:\n\u001b[1;32m    245\u001b[0m     \u001b[38;5;66;03m# open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)\u001b[39;00m\n\u001b[1;32m    246\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(path, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrb\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m--> 247\u001b[0m         img \u001b[38;5;241m=\u001b[39m \u001b[43mImage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    248\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m img\u001b[38;5;241m.\u001b[39mconvert(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRGB\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n","File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/PIL/Image.py:3298\u001b[0m, in \u001b[0;36mopen\u001b[0;34m(fp, mode, formats)\u001b[0m\n\u001b[1;32m   3291\u001b[0m     checked_formats \u001b[38;5;241m=\u001b[39m formats\u001b[38;5;241m.\u001b[39mcopy()\n\u001b[1;32m   3292\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m init():\n\u001b[1;32m   3293\u001b[0m         im \u001b[38;5;241m=\u001b[39m _open_core(\n\u001b[1;32m   3294\u001b[0m             fp,\n\u001b[1;32m   3295\u001b[0m             filename,\n\u001b[1;32m   3296\u001b[0m             prefix,\n\u001b[1;32m   3297\u001b[0m             \u001b[38;5;28mtuple\u001b[39m(\u001b[38;5;28mformat\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m \u001b[38;5;28mformat\u001b[39m \u001b[38;5;129;01min\u001b[39;00m formats \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mformat\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m checked_formats),\n\u001b[0;32m-> 3298\u001b[0m         )\n\u001b[1;32m   3300\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m im:\n\u001b[1;32m   3301\u001b[0m     im\u001b[38;5;241m.\u001b[39m_exclusive_fp \u001b[38;5;241m=\u001b[39m exclusive_fp\n","\u001b[0;31mUnidentifiedImageError\u001b[0m: cannot identify image file <_io.BufferedReader name='/kaggle/input/11-785-fall-20-homework-2-part-2/classification_data/val_data/n002213/0045_01.jpg'>"],"ename":"UnidentifiedImageError","evalue":"cannot identify image file <_io.BufferedReader name='/kaggle/input/11-785-fall-20-homework-2-part-2/classification_data/val_data/n002213/0045_01.jpg'>","output_type":"error"}]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nmean = np.array([0.485, 0.456, 0.406])\nstd =  np.array([0.229, 0.224, 0.225])","metadata":{"execution":{"iopub.status.busy":"2024-05-18T07:53:32.929255Z","iopub.status.idle":"2024-05-18T07:53:32.929619Z","shell.execute_reply.started":"2024-05-18T07:53:32.92945Z","shell.execute_reply":"2024-05-18T07:53:32.929465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X.squeeze(0).cpu().numpy().transpose((1,2,0)) * std + mean)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T07:53:32.930824Z","iopub.status.idle":"2024-05-18T07:53:32.931147Z","shell.execute_reply.started":"2024-05-18T07:53:32.930981Z","shell.execute_reply":"2024-05-18T07:53:32.930994Z"},"trusted":true},"execution_count":null,"outputs":[]}]}