{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"},{"sourceId":7430862,"sourceType":"datasetVersion","datasetId":4324239},{"sourceId":7430974,"sourceType":"datasetVersion","datasetId":4324320}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle link: https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition","metadata":{}},{"cell_type":"code","source":"# RUN THESE IF THE ENVIRONMENT DOESN'T WORK\n# MAY REQUIRE RESTARTING KERNEL\n#!pip uninstall --yes torch torchvision torchaudio\n#!pip install cuda-python==11.8.0\n#!pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118\n!pip install imagesize","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:09.610330Z","iopub.execute_input":"2024-01-18T21:29:09.610719Z","iopub.status.idle":"2024-01-18T21:29:24.460111Z","shell.execute_reply.started":"2024-01-18T21:29:09.610687Z","shell.execute_reply":"2024-01-18T21:29:24.458669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run the above cell if this doesn't output True\nimport torch\ntorch.cuda.is_available()","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:24.462362Z","iopub.execute_input":"2024-01-18T21:29:24.462839Z","iopub.status.idle":"2024-01-18T21:29:27.807118Z","shell.execute_reply.started":"2024-01-18T21:29:24.462799Z","shell.execute_reply":"2024-01-18T21:29:27.806097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\nif not (os.path.exists(\"/kaggle/working/model\") and os.path.isdir(\"/kaggle/working/model\")):\n    os.mkdir(\"/kaggle/working/model\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-18T21:29:27.808476Z","iopub.execute_input":"2024-01-18T21:29:27.809022Z","iopub.status.idle":"2024-01-18T21:29:28.266942Z","shell.execute_reply.started":"2024-01-18T21:29:27.808983Z","shell.execute_reply":"2024-01-18T21:29:28.265879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport glob\nimport imagesize\nimport time\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport math\nimport wandb\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import log_loss\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets\nfrom torchvision.io import read_image\nfrom torchvision.transforms import v2 as transforms\nfrom torchinfo import summary\n\nfrom tqdm import tqdm\nimport ipywidgets as widgets\nfrom ipywidgets import interact\n\nconfig = {\n    \"width\": 200, # 260 is default, updates to this require changing input dimension in fully connected layer of model function\n    \"height\": 200, # 260 is default, updates to this require changing input dimension in fully connected layer of model function\n    \"img_channels\": 3,\n    \"transform_mode\": \"regular\", # scale, regular\n    \"rotation\": 90, # 90 is default, controls random rotations in the transforms\n    \"epochs\": 10,\n    \"lr\": 0.01, # 0.01 is default\n    \"batch_size\": 64, # 64 is default\n    \"optimizer\": \"SGD\", # SGD, adam\n    \"decay\": 0.01, # 0.01 is default, decay for adam optimizer\n    \"momentum\": 0.9, # 0.9 is default, SGD momentum\n    \"architecture\": [(1, 64), (1, 128), (2, 256), (2, 512), (2, 512)], # (n_convolutions, output_channels)\n    \"net\": {\n        \"kernel_size\": 3, # 3 is default\n        \"stride\": 1, # 1 is default\n        \"kernel_progression\": 0, # 0 is default, defines whether the convolutions should be getting bigger/smaller (with negative values)\n        \"stride_progression\": 0, # 0 is default, defines whether the stride should increase/decrease (with negative values)\n        \"padding\": \"same\",\n        \"conv_dropout\": 0.1, # 0.1 is default, dropout for convolution layers\n        \"pooling\": \"max\", # max, avg\n        \"pool_size\": 2, # 2 is default, kernel size for pooling operations\n        \"minimum_k\": 3, # 3 is default, when shrinking kernel, this is the minimum (kernel sizes below 1 make no sense)\n        \"minimum_s\": 1, # 1 is default, when reducing stride, this is the minimum (strides below 1 make no sense)\n        \"activation\": \"leaky_relu\", # relu, leaky_relu\n        \"slope\": 1e-2, # 1e-2 is default\n        \"initialization\": \"default\" # kaiming, default\n    },\n    \"criterion\": \"BCE\", # BCE, MSE\n    \"logging\": \"standard\", # standard, debug\n    \"compute_stats\": False,\n    \"submit\": True\n}\n\nrun = wandb.init(project=\"MLDL_CatsVsDogs\",\n                 name=f\"VGG-11 Without Normalization, {config['width']}x{config['height']}px, {config['epochs']} Epochs\",\n                 notes=\"Doing a very bad model to test submission scoring because it looks like there's something wrong with it.\",\n                 mode=\"online\" if not config[\"submit\"] else \"disabled\",\n                 config=config,\n                 save_code=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:28.269534Z","iopub.execute_input":"2024-01-18T21:29:28.270026Z","iopub.status.idle":"2024-01-18T21:29:30.919267Z","shell.execute_reply.started":"2024-01-18T21:29:28.269988Z","shell.execute_reply":"2024-01-18T21:29:30.918323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"train_dir = \"train\"\ntest_dir = \"test\"\n\nif not (os.path.exists(train_dir) and os.path.isdir(train_dir)):\n    with zipfile.ZipFile(\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip\") as train_zip:\n        train_zip.extractall(\"\")\n    \nif not (os.path.exists(test_dir) and os.path.isdir(test_dir)):\n    with zipfile.ZipFile(\"/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip\") as test_zip:\n        test_zip.extractall(\"\")\n    \ntrain_list = glob.glob(os.path.join(train_dir,\"*.jpg\"))\ntest_list = glob.glob(os.path.join(test_dir, \"*.jpg\"))\nprint(f\"Train Data: {len(train_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:30.920640Z","iopub.execute_input":"2024-01-18T21:29:30.921140Z","iopub.status.idle":"2024-01-18T21:29:47.183462Z","shell.execute_reply.started":"2024-01-18T21:29:30.921109Z","shell.execute_reply":"2024-01-18T21:29:47.182344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [path.split(\"/\")[-1].split(\".\")[0] for path in train_list]","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:47.184810Z","iopub.execute_input":"2024-01-18T21:29:47.185223Z","iopub.status.idle":"2024-01-18T21:29:47.205595Z","shell.execute_reply.started":"2024-01-18T21:29:47.185188Z","shell.execute_reply":"2024-01-18T21:29:47.204537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot random image with their label","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1, len(train_list), size=9)\nfig, axes = plt.subplots(3, 3, figsize=(16, 12))\n\nfor idx, ax in enumerate(axes.ravel()):\n    img = Image.open(train_list[idx])\n    ax.set_title(labels[idx])\n    ax.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:47.206762Z","iopub.execute_input":"2024-01-18T21:29:47.207108Z","iopub.status.idle":"2024-01-18T21:29:50.269813Z","shell.execute_reply.started":"2024-01-18T21:29:47.207081Z","shell.execute_reply":"2024-01-18T21:29:50.268440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Use Sklearn to split the training data","metadata":{}},{"cell_type":"code","source":"train_list, valid_list = train_test_split(train_list, \n                                          test_size=0.2,\n                                          stratify=labels)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:50.271361Z","iopub.execute_input":"2024-01-18T21:29:50.271700Z","iopub.status.idle":"2024-01-18T21:29:50.305582Z","shell.execute_reply.started":"2024-01-18T21:29:50.271673Z","shell.execute_reply":"2024-01-18T21:29:50.304706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train Data: {len(train_list)}\")\nprint(f\"Validation Data: {len(valid_list)}\")\nprint(f\"Test Data: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:50.306749Z","iopub.execute_input":"2024-01-18T21:29:50.307400Z","iopub.status.idle":"2024-01-18T21:29:50.312875Z","shell.execute_reply.started":"2024-01-18T21:29:50.307370Z","shell.execute_reply":"2024-01-18T21:29:50.311455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metas = {}\nfor i, img in enumerate(train_list):\n    metas[i] = imagesize.get(img) \ndf = pd.DataFrame.from_dict([metas]).T.reset_index().drop(\"index\", axis=1).set_axis([\"Size\"], axis=\"columns\")\ndf[[\"Width\", \"Height\"]] = pd.DataFrame(df[\"Size\"].tolist(), index=df.index)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:50.316955Z","iopub.execute_input":"2024-01-18T21:29:50.317279Z","iopub.status.idle":"2024-01-18T21:29:51.881384Z","shell.execute_reply.started":"2024-01-18T21:29:50.317239Z","shell.execute_reply":"2024-01-18T21:29:51.880000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:51.882638Z","iopub.execute_input":"2024-01-18T21:29:51.884243Z","iopub.status.idle":"2024-01-18T21:29:51.907824Z","shell.execute_reply.started":"2024-01-18T21:29:51.884210Z","shell.execute_reply":"2024-01-18T21:29:51.906704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NO SCALING/NORMALIZATION\ntrain_transforms = transforms.Compose([\n    # TODO: define traininig augmentations \n    transforms.Resize((config[\"width\"], config[\"height\"]), antialias=True),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.RandomRotation(config[\"rotation\"]),\n    transforms.ToDtype(torch.float32)\n])\n\ntest_transforms = transforms.Compose([\n    # TODO: define test augmentations\n    transforms.Resize((config[\"width\"], config[\"height\"]), antialias=True),\n    transforms.ToDtype(torch.float32)\n])\n\n# For visualization purposes\ninverse_transforms = transforms.Compose([\n    transforms.ToDtype(torch.uint8),\n    transforms.ToPILImage()\n])\n\n# Scaling and normalization\ntrain_transforms_scaled = transforms.Compose([\n    transforms.Resize((config[\"width\"], config[\"height\"]), antialias=True),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.RandomRotation(config[\"rotation\"]),\n    transforms.ToDtype(torch.float32),\n    transforms.Normalize(mean=[124.3542, 115.8128, 106.1118], # Computed means for full scale train dataset\n                         std=[66.3718, 64.6729, 65.3108])     # Computed stds for full scale train dataset\n    # transforms.Normalize(mean=[0.4877, 0.4542, 0.4161], # Computed means for scaled train dataset\n    #                      std=[0.2603, 0.2536, 0.2561])  # Computed stds for scaled train dataset\n])\n\ntest_transforms_scaled = transforms.Compose([\n    transforms.Resize((config[\"width\"], config[\"height\"]), antialias=True),\n    transforms.ToDtype(torch.float32),\n    transforms.Normalize(mean=[124.3542, 115.8128, 106.1118], # Computed means for full scale train dataset\n                         std=[66.3718, 64.6729, 65.3108])     # Computed stds for full scale train dataset\n    # transforms.Normalize(mean=[0.4877, 0.4542, 0.4161], # Computed means for scaled train dataset\n    #                      std=[0.2603, 0.2536, 0.2561])  # Computed stds for scaled train dataset\n])\n\n# For visualization purposes\ninverse_transforms_scaled = transforms.Compose([\n    transforms.Normalize(mean=[0., 0., 0.],\n                         std=[1/66.3718, 1/64.6729, 1/65.3108]),\n    transforms.Normalize(mean=[-124.3542, -115.8128, -106.1118],\n                         std=[1., 1., 1.]),\n    # transforms.Normalize(mean=[0., 0., 0.],\n    #                      std=[1/0.2603, 1/0.2536, 1/0.2561]),\n    # transforms.Normalize(mean=[-0.4877, -0.4542, -0.4161],\n    #                     std=[1., 1., 1.]),\n    transforms.ToDtype(dtype=torch.uint8),\n    transforms.ToPILImage()\n])","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:51.911691Z","iopub.execute_input":"2024-01-18T21:29:51.912063Z","iopub.status.idle":"2024-01-18T21:29:51.927257Z","shell.execute_reply.started":"2024-01-18T21:29:51.912032Z","shell.execute_reply":"2024-01-18T21:29:51.926222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    \n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n        self.filelength = len(file_list)\n\n    def __len__(self):\n        return self.filelength\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = read_image(img_path)\n        img_transformed = self.transform(img)\n        label = img_path.split(\"/\")[-1].split(\".\")[0]\n        label = 1.0 if label == \"dog\" else 0.0\n        return img_transformed, torch.FloatTensor([label])","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:51.928628Z","iopub.execute_input":"2024-01-18T21:29:51.928998Z","iopub.status.idle":"2024-01-18T21:29:51.940375Z","shell.execute_reply.started":"2024-01-18T21:29:51.928961Z","shell.execute_reply":"2024-01-18T21:29:51.939383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = CatsDogsDataset(train_list, transform=train_transforms)\nvalid_data = CatsDogsDataset(valid_list, transform=test_transforms)\ntest_data = CatsDogsDataset(test_list, transform=test_transforms)\n\ntrain_data_scale = CatsDogsDataset(train_list, transform=train_transforms_scaled)\nvalid_data_scale = CatsDogsDataset(valid_list, transform=test_transforms_scaled)\ntest_data_scale = CatsDogsDataset(test_list, transform=test_transforms_scaled)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:51.941762Z","iopub.execute_input":"2024-01-18T21:29:51.942276Z","iopub.status.idle":"2024-01-18T21:29:51.954123Z","shell.execute_reply.started":"2024-01-18T21:29:51.942232Z","shell.execute_reply":"2024-01-18T21:29:51.952921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(labels, bins=2, histtype=\"barstacked\", stacked=True, label=\"Training samples\");","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:51.955888Z","iopub.execute_input":"2024-01-18T21:29:51.956298Z","iopub.status.idle":"2024-01-18T21:29:52.302923Z","shell.execute_reply.started":"2024-01-18T21:29:51.956259Z","shell.execute_reply":"2024-01-18T21:29:52.301594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"workers = os.cpu_count()\n\ntrain_loader = DataLoader(dataset=train_data, batch_size=config[\"batch_size\"], shuffle=True, num_workers=workers)\nval_loader = DataLoader(dataset=valid_data, batch_size=config[\"batch_size\"], shuffle=False, num_workers=workers)\ntest_loader = DataLoader(dataset=test_data, batch_size=config[\"batch_size\"], shuffle=False, num_workers=workers)\n\ntrain_loader_scale = DataLoader(dataset=train_data_scale, batch_size=config[\"batch_size\"], shuffle=True, num_workers=workers)\nval_loader_scale = DataLoader(dataset=valid_data_scale, batch_size=config[\"batch_size\"], shuffle=False, num_workers=workers)\ntest_loader_scale = DataLoader(dataset=test_data_scale, batch_size=config[\"batch_size\"], shuffle=False, num_workers=workers)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:52.304496Z","iopub.execute_input":"2024-01-18T21:29:52.304950Z","iopub.status.idle":"2024-01-18T21:29:52.314183Z","shell.execute_reply.started":"2024-01-18T21:29:52.304909Z","shell.execute_reply":"2024-01-18T21:29:52.313118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_pixel_stats(loader):\n    psum = 0\n    psum_sq = 0\n    psum_r = 0\n    psum_g = 0\n    psum_b = 0\n    psum_r_sq = 0\n    psum_g_sq = 0\n    psum_b_sq = 0\n    for (imgs, _) in tqdm(loader):\n        psum = imgs.sum(dim=(2,3))\n        psum_sq = (imgs**2).sum(dim=(2,3))\n        psum_r += psum[:, 0].sum()\n        psum_g += psum[:, 1].sum()\n        psum_b += psum[:, 2].sum()\n        psum_r_sq += psum_sq[:, 0].sum()\n        psum_g_sq += psum_sq[:, 1].sum()\n        psum_b_sq += psum_sq[:, 2].sum()\n\n    count = len(train_data) * config[\"width\"] * config[\"height\"]\n\n    # mean and STD\n    red_mean = psum_r / count\n    green_mean = psum_g / count\n    blue_mean = psum_b / count\n\n    red_var  = (psum_r_sq / count) - (red_mean ** 2)\n    red_std  = torch.sqrt(red_var)\n    green_var = (psum_g_sq / count) - (green_mean ** 2)\n    green_std = torch.sqrt(green_var)\n    blue_var = (psum_b_sq / count) - (blue_mean ** 2)\n    blue_std = torch.sqrt(blue_var)\n\n    # output\n    print(\"Training data stats:\")\n    print(f\"- mean: [{red_mean.item():.4f}, {green_mean.item():.4f}, {blue_mean.item():.4f}]\")\n    print(f\"- std:  [{red_std.item():.4f}, {green_std.item():.4f}, {blue_std.item():.4f}]\")","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:52.315749Z","iopub.execute_input":"2024-01-18T21:29:52.316438Z","iopub.status.idle":"2024-01-18T21:29:52.331646Z","shell.execute_reply.started":"2024-01-18T21:29:52.316398Z","shell.execute_reply":"2024-01-18T21:29:52.330357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config[\"compute_stats\"]:\n    compute_pixel_stats(train_loader)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:29:52.333226Z","iopub.execute_input":"2024-01-18T21:29:52.334225Z","iopub.status.idle":"2024-01-18T21:30:53.343250Z","shell.execute_reply.started":"2024-01-18T21:29:52.334182Z","shell.execute_reply":"2024-01-18T21:30:53.342090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config[\"compute_stats\"]:\n    compute_pixel_stats(train_loader_scale)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:30:53.345194Z","iopub.execute_input":"2024-01-18T21:30:53.346044Z","iopub.status.idle":"2024-01-18T21:31:54.237696Z","shell.execute_reply.started":"2024-01-18T21:30:53.345983Z","shell.execute_reply":"2024-01-18T21:31:54.236493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check what a normalized image looks like after the composed transformations\nif not config[\"submit\"]:\n    @interact(img=(0,100,1))\n    def inspect_scaled_image(img=1):\n        pil = inverse_transforms_scaled(train_data_scale[img][0])\n        plt.imshow(pil)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:54.239362Z","iopub.execute_input":"2024-01-18T21:31:54.239686Z","iopub.status.idle":"2024-01-18T21:31:54.660746Z","shell.execute_reply.started":"2024-01-18T21:31:54.239656Z","shell.execute_reply":"2024-01-18T21:31:54.659787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check what an image looks like after the composed transformations\nif not config[\"submit\"]:\n    @interact(img=(0,100,1))\n    def inspect_image(img=1):\n        pil = inverse_transforms(train_data[img][0])\n        plt.imshow(pil)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:54.662266Z","iopub.execute_input":"2024-01-18T21:31:54.662624Z","iopub.status.idle":"2024-01-18T21:31:55.000235Z","shell.execute_reply.started":"2024-01-18T21:31:54.662593Z","shell.execute_reply":"2024-01-18T21:31:54.999182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config[\"transform_mode\"] == \"scale\":\n    train_loader = train_loader_scale\n    val_loader = val_loader_scale\n    test_loader = test_loader_scale","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:55.001564Z","iopub.execute_input":"2024-01-18T21:31:55.001903Z","iopub.status.idle":"2024-01-18T21:31:55.007707Z","shell.execute_reply.started":"2024-01-18T21:31:55.001850Z","shell.execute_reply":"2024-01-18T21:31:55.006722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Old network definition, non-modular, clunky\nclass Net(nn.Module):\n    \n    def __init__(self, kernel_size=5, kernel_progression=0, stride=1, stride_progression=0, padding=\"same\", ch1=10, ch2=32, ch3=64, ch4=128, pooling=\"max\", pool_size=2, *, minimum_k=2, minimum_s=1):\n        super(Net, self).__init__()\n        self.conv1 = nn.Conv2d(3, ch1, kernel_size, stride, padding)\n        self.conv2 = nn.Conv2d(ch1, ch2,\n                               max(kernel_size+kernel_progression, minimum_k),\n                               max(stride+stride_progression, minimum_s),\n                               padding)\n        self.conv3 = nn.Conv2d(ch2, ch3,\n                               max(kernel_size+2*kernel_progression, minimum_k),\n                               max(stride+2*stride_progression, minimum_s),\n                               padding)\n        self.conv4 = nn.Conv2d(ch3, ch4,\n                               max(kernel_size+3*kernel_progression, minimum_k),\n                               max(stride+3*stride_progression, minimum_s),\n                               padding)\n        self.pool = nn.AvgPool2d(pool_size) if pooling == \"avg\" else nn.MaxPool2d(pool_size)\n        self.flat = nn.Flatten()\n        self.fc1 = nn.Linear(36864, 4096)\n        self.fc2 = nn.Linear(4096, 128)\n        self.fc3 = nn.Linear(128, 1)\n        \n    def forward(self, x):\n        x = self.pool(F.relu(self.conv1(x)))\n        if config[\"logging\"] == \"verbose\":\n            print(f\"FIRST STEP: {x.shape}\")\n        x = self.pool(F.relu(self.conv2(x)))\n        if config[\"logging\"] == \"verbose\":\n            print(f\"SECOND STEP: {x.shape}\")\n        x = self.pool(F.relu(self.conv3(x)))\n        if config[\"logging\"] == \"verbose\":\n            print(f\"THIRD STEP: {x.shape}\")\n        x = self.pool(F.relu(self.conv4(x)))\n        if config[\"logging\"] == \"verbose\":\n            print(f\"FOURTH STEP: {x.shape}\")\n        x = self.flat(x)\n        if config[\"logging\"] == \"verbose\":\n            print(f\"RESHAPE: {x.shape}\")\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        if config[\"criterion\"] == \"MSE\":\n            return F.sigmoid(self.fc3(x))\n        else:\n            return self.fc3(x)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:55.009425Z","iopub.execute_input":"2024-01-18T21:31:55.009752Z","iopub.status.idle":"2024-01-18T21:31:55.025186Z","shell.execute_reply.started":"2024-01-18T21:31:55.009723Z","shell.execute_reply":"2024-01-18T21:31:55.024221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# New and improved network definition\nfrom typing import List, Dict\n\n# Custom kaiming initialization kills gradient for some reason\ndef kaiming_initialization(layer, slope=1e-2, nonlinearity=\"relu\"):\n    with torch.no_grad():\n        nn.init.kaiming_normal_(layer.weight, a=slope, nonlinearity=nonlinearity)\n        (fan_in, fan_out) = nn.init._calculate_fan_in_and_fan_out(layer.weight)\n        bound = 1 / math.sqrt(fan_in)\n        nn.init.uniform_(layer.bias, -bound, bound)\n    return layer\n\ndef build_conv_block(in_channels, out_channels, convs: List[Dict], pool: Dict, dropout = 0, initialization = \"default\", activation = \"relu\", slope = 1e-2):\n    \n    layers = []\n    for params in convs:\n        if initialization == \"kaiming\":\n            layers.append(kaiming_initialization(nn.Conv2d(in_channels, out_channels, **params), slope=slope, nonlinearity=activation))\n        else:\n            layers.append(nn.Conv2d(in_channels, out_channels, **params))\n        if activation == \"leaky_relu\":\n            layers.append(nn.LeakyReLU(slope))\n        else:\n            layers.append(nn.ReLU())\n        in_channels = out_channels\n    \n    if pool.pop(\"type\", \"max\") == \"avg\":\n        layers.append(nn.AvgPool2d(**pool))\n    else:\n        layers.append(nn.MaxPool2d(**pool))\n        \n    if dropout > 0:\n        layers.append(nn.Dropout(dropout))\n    \n    return nn.Sequential(*layers)\n\ndef vgg(ch_start, arch, config):\n    blocks = []\n    ch_in = ch_start\n    for i, (convs, ch_out) in enumerate(arch):\n        kernel_size = max(config[\"kernel_size\"] + i*config[\"kernel_progression\"], config[\"minimum_k\"])\n        stride = max(config[\"stride\"] + i*config[\"stride_progression\"], config[\"minimum_s\"])\n        padding = config[\"padding\"]\n        params = [{\"kernel_size\": kernel_size, \"stride\": stride, \"padding\": padding}] * convs\n        pool = {\"kernel_size\": config[\"pool_size\"], \"type\": config[\"pooling\"]}\n        blocks.append(build_conv_block(ch_in, ch_out, params, pool, config[\"conv_dropout\"], config[\"initialization\"], config[\"activation\"], config[\"slope\"]))\n        ch_in = ch_out\n    \n    # The input dimension has to be set manually for Linear layers, which is annoying\n    # LazyLinear allows for automatic inference of input dimension from first forward pass\n    # but is still in early experimental beta and not recommended.\n    # One option is to compute the input dimension by keeping track of all the previous\n    # layers and their impact on matrix size.\n    # However here for simplicity we set an arbitrary number and run the training,\n    # when we get the expected failure we can read the actual input size from the error.\n    # input_channels = 32768 # 260x260px\n    input_channels = 18432 # 200x200px\n    if config[\"initialization\"] == \"kaiming\":\n        fc1 = kaiming_initialization(nn.Linear(input_channels, 4096), config[\"slope\"], config[\"activation\"])\n        fc2 = kaiming_initialization(nn.Linear(4096, 128), config[\"slope\"], config[\"activation\"])\n        fc3 = kaiming_initialization(nn.Linear(128, 1), config[\"slope\"], config[\"activation\"])\n    else:\n        fc1 = nn.Linear(input_channels, 4096)\n        fc2 = nn.Linear(4096, 128)\n        fc3 = nn.Linear(128, 1)\n    \n    net = nn.Sequential(*blocks, nn.Flatten(),\n                        fc1, nn.ReLU() if config[\"activation\"] == \"relu\" else nn.LeakyReLU(config[\"slope\"]), nn.Dropout(0.5),\n                        fc2, nn.ReLU() if config[\"activation\"] == \"relu\" else nn.LeakyReLU(config[\"slope\"]), nn.Dropout(0.5),\n                        fc3)\n    \n    return net","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:55.026426Z","iopub.execute_input":"2024-01-18T21:31:55.026726Z","iopub.status.idle":"2024-01-18T21:31:55.049812Z","shell.execute_reply.started":"2024-01-18T21:31:55.026700Z","shell.execute_reply":"2024-01-18T21:31:55.048738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inspect_grad(model, output=False):\n    layers = []\n    names = [\"Conv1\", \"Conv2\", \"Conv3.1\", \"Conv3.2\",\n             \"Conv4.1\", \"Conv4.2\", \"Conv5.1\", \"Conv5.2\",\n             \"Linear1\", \"Linear2\", \"Linear3\"]\n    for name, layer in model.named_parameters():\n        if name.endswith(\"weight\"):\n            layers.append(layer)\n\n    means = []\n    maxes = []\n    names.reverse()\n    layers.reverse()\n    for name, layer in zip(names, layers):\n        if output:\n            print(name)\n            print(layer.grad)\n        mean = layer.grad.mean().item()\n        mx = layer.grad.max().item()\n        means.append(mean)\n        maxes.append(mx)\n        if output:\n            print(f\"{name}:\")\n            print(f\"- mean: {mean}\")\n            print(f\"- max: {mx}\")\n        \n    return max(means), sum(maxes) / len(maxes)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:55.051024Z","iopub.execute_input":"2024-01-18T21:31:55.051353Z","iopub.status.idle":"2024-01-18T21:31:55.065816Z","shell.execute_reply.started":"2024-01-18T21:31:55.051325Z","shell.execute_reply":"2024-01-18T21:31:55.064815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = vgg(config[\"img_channels\"], config[\"architecture\"], config[\"net\"])\nmodel = model.to(device)\nsummary(model, input_size=(config[\"batch_size\"], config[\"img_channels\"], config[\"width\"], config[\"height\"]))","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:55.067187Z","iopub.execute_input":"2024-01-18T21:31:55.067851Z","iopub.status.idle":"2024-01-18T21:31:56.874352Z","shell.execute_reply.started":"2024-01-18T21:31:55.067814Z","shell.execute_reply":"2024-01-18T21:31:56.873252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config[\"logging\"] == \"debug\":\n    for name, layer in model.named_parameters():\n        print(name)\n        print(layer)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:56.875671Z","iopub.execute_input":"2024-01-18T21:31:56.876029Z","iopub.status.idle":"2024-01-18T21:31:56.881240Z","shell.execute_reply.started":"2024-01-18T21:31:56.875999Z","shell.execute_reply":"2024-01-18T21:31:56.880242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config[\"optimizer\"] == \"adam\":\n    optim = torch.optim.Adam(model.parameters(), lr=config[\"lr\"], weight_decay=config[\"decay\"])\nelif config[\"optimizer\"] == \"SGD\":\n    optim = torch.optim.SGD(model.parameters(), lr=config[\"lr\"], momentum=config[\"momentum\"])\nelse:\n    optim = torch.optim.SGD(model.parameters(), lr=config[\"lr\"])\n\nif config[\"criterion\"] == \"MSE\":\n    criterion = nn.MSELoss()\nelse:\n    criterion = nn.BCEWithLogitsLoss()\n\nstart_time = time.time()\nwith tqdm(total=config[\"epochs\"], position=1, leave=True) as pbar:\n    for epoch in range(config[\"epochs\"]):\n\n        running_loss = 0.0\n        running_acc = 0.0\n        model.train()\n        for i, (imgs, labels) in enumerate(tqdm(train_loader, position=0, leave=True)):\n            optim.zero_grad()\n            imgs, labels = imgs.to(device), labels.to(device)\n            outputs = model(imgs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optim.step()\n            with torch.no_grad():\n                current_loss = loss.cpu().item()\n                running_loss += current_loss\n                predictions = torch.where(F.sigmoid(outputs) > 0.5, 1.0, 0.0)\n                current_acc = (predictions == labels).cpu().float().mean().item()\n                running_acc += current_acc\n                if \"verbose\" in config[\"logging\"].split(\",\"):\n                    wandb.log({\"training\": {\"pointwise_loss\": current_loss,\n                                            \"pointwise_acc\": current_acc,\n                                            \"point\": i}}, commit=False)\n        wandb.log({\"training\": {\"loss\": running_loss / (i+1),\n                                \"accuracy\": running_acc / (i+1)},\n                   \"epoch\": epoch+1}, commit=False)\n        \n        max_of_means, mean_of_maxes = inspect_grad(model, output=False if config[\"logging\"] != \"debug\" else True)\n        wandb.log({\"max_mean_gradient\": max_of_means,\n                   \"average_max_gradient\": mean_of_maxes}, commit=False)\n            \n        running_loss = 0.0\n        running_acc = 0.0\n        running_log_loss = 0.0\n        model.eval()\n        with torch.no_grad():\n            for i, (imgs, labels) in enumerate(tqdm(val_loader, position=0, leave=True)):\n                imgs, labels = imgs.to(device), labels.to(device)\n                outputs = model(imgs)\n                loss = criterion(outputs, labels)\n                current_loss = loss.cpu().item()\n                running_loss += current_loss\n                normalized_outputs = F.sigmoid(outputs)\n                predictions = torch.where(normalized_outputs > 0.5, 1.0, 0.0)\n                current_acc = (predictions == labels).cpu().float().mean().item()\n                running_acc += current_acc\n                current_log_loss = log_loss(labels.cpu(), normalized_outputs.cpu(), normalize=False)\n                running_log_loss += current_log_loss\n                if \"verbose\" in config[\"logging\"].split(\",\"):\n                    wandb.log({\"validation\": {\"pointwise_loss\": current_loss,\n                                              \"pointwise_acc\": current_acc,\n                                              \"pointwise_log_loss\": current_log_loss,\n                                              \"point\": i}}, commit=False)\n        wandb.log({\"validation\": {\"loss\": running_loss / (i+1),\n                                  \"log_loss\": running_log_loss / len(val_loader.dataset),\n                                  \"accuracy\": running_acc / (i+1)},\n                   \"epoch\": epoch+1}, commit=True)\n        torch.save(model.state_dict(), f\"/kaggle/working/model/model.pt\")\n        pbar.update()\n    \nwandb.log({\"total_training_time\": time.time() - start_time})","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:31:56.888255Z","iopub.execute_input":"2024-01-18T21:31:56.888573Z","iopub.status.idle":"2024-01-18T21:46:41.735635Z","shell.execute_reply.started":"2024-01-18T21:31:56.888546Z","shell.execute_reply":"2024-01-18T21:46:41.734175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config[\"submit\"]:\n    model.eval()\n    with torch.no_grad():\n        preds = torch.Tensor().to(device)\n        for imgs in tqdm(test_loader):\n            output = F.sigmoid(model(imgs[0].to(device)))\n            # output = torch.where(F.sigmoid(model(imgs[0].to(device))) > 0.5, 1.0, 0.0)\n            preds = torch.cat((preds, output))\n    out_df = pd.DataFrame(np.c_[np.arange(1, len(test_data)+1)[:,None], preds.cpu().numpy()], columns=[\"id\", \"label\"])\n    out_df[\"id\"] = out_df[\"id\"].astype(int)\n    out_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:46:41.737340Z","iopub.execute_input":"2024-01-18T21:46:41.737709Z","iopub.status.idle":"2024-01-18T21:47:05.741190Z","shell.execute_reply.started":"2024-01-18T21:46:41.737671Z","shell.execute_reply":"2024-01-18T21:47:05.739650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_path = \"/kaggle/input/sonnuriu/sonnuriu_1.jpeg\"\nimg = read_image(img_path)\nimg1 = test_transforms(img)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T22:31:12.883285Z","iopub.execute_input":"2024-01-18T22:31:12.884360Z","iopub.status.idle":"2024-01-18T22:31:12.909709Z","shell.execute_reply.started":"2024-01-18T22:31:12.884317Z","shell.execute_reply":"2024-01-18T22:31:12.908453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_path = \"/kaggle/input/sonnuriu/sonnuriu_2.jpeg\"\nimg = read_image(img_path)\nimg2 = test_transforms(img)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T22:31:13.610216Z","iopub.execute_input":"2024-01-18T22:31:13.610645Z","iopub.status.idle":"2024-01-18T22:31:13.690811Z","shell.execute_reply.started":"2024-01-18T22:31:13.610610Z","shell.execute_reply":"2024-01-18T22:31:13.689740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nwith torch.no_grad():\n    model = model.cpu()\n    fake_batch = torch.zeros(64, 3, 200, 200)\n    fake_batch[0], fake_batch[1] = img1, img2\n    output = model(fake_batch)","metadata":{"execution":{"iopub.status.busy":"2024-01-18T22:31:14.589238Z","iopub.execute_input":"2024-01-18T22:31:14.589716Z","iopub.status.idle":"2024-01-18T22:31:22.003618Z","shell.execute_reply.started":"2024-01-18T22:31:14.589679Z","shell.execute_reply":"2024-01-18T22:31:22.002356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img1_pred = F.sigmoid(output)[0].item()\nimg2_pred = F.sigmoid(output)[1].item()","metadata":{"execution":{"iopub.status.busy":"2024-01-18T22:31:22.005644Z","iopub.execute_input":"2024-01-18T22:31:22.006052Z","iopub.status.idle":"2024-01-18T22:31:22.012217Z","shell.execute_reply.started":"2024-01-18T22:31:22.006019Z","shell.execute_reply":"2024-01-18T22:31:22.011108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fsize = 40\nfig, axes = plt.subplots(1, 2, figsize=(32, 32))\nax1 = axes.ravel()[0]\nax2 = axes.ravel()[1]\nax1.set_title(f\"This is most likely a {'dog' if img1_pred > 0.5 else 'cat'} ({img1_pred:.2f})\", fontsize=fsize)\nax2.set_title(f\"This is most likely a {'dog' if img2_pred > 0.5 else 'cat'} ({img2_pred:.2f})\", fontsize=fsize)\nax1.imshow(inverse_transforms(img1))\nax2.imshow(inverse_transforms(img2))","metadata":{"execution":{"iopub.status.busy":"2024-01-18T22:31:22.014833Z","iopub.execute_input":"2024-01-18T22:31:22.015531Z","iopub.status.idle":"2024-01-18T22:31:23.454551Z","shell.execute_reply.started":"2024-01-18T22:31:22.015485Z","shell.execute_reply":"2024-01-18T22:31:23.453116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2024-01-18T21:47:05.743263Z","iopub.execute_input":"2024-01-18T21:47:05.743678Z","iopub.status.idle":"2024-01-18T21:47:05.749736Z","shell.execute_reply.started":"2024-01-18T21:47:05.743637Z","shell.execute_reply":"2024-01-18T21:47:05.748586Z"},"trusted":true},"execution_count":null,"outputs":[]}]}