{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Simple Dataset reading pipeline\nDataset is already downloaded. Hence this read takes <1 minutes compared to 20+ minutes from huggingface datasets","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision, torchinfo, torchmetrics\nimport torchvision\nfrom sklearn.model_selection import train_test_split\nimport os, glob, zipfile\nfrom tqdm import tqdm\nfrom PIL import Image\n\ndef DOWNLOAD_DATASETS():\n    zip_files = ['test.zip', 'train.zip']\n\n    for zip_file in zip_files:\n        with zipfile.ZipFile(\"../input/dogs-vs-cats-redux-kernels-edition/{}\".format(zip_file),\"r\") as z:\n            z.extractall(\".\")\n            print(\"{} unzipped\".format(zip_file))\n\n    train_file_names_list = glob.glob(os.path.join(\"../working/train\",'*.jpg'))\n    test_file_names_list  = glob.glob(os.path.join(\"../working/test\", '*.jpg'))\n\n    train_list, val_list  = train_test_split(train_file_names_list, test_size=0.2)\n\n    transformation_list =  torchvision.transforms.Compose([\n            torchvision.transforms.Resize((224, 224)),\n            torchvision.transforms.ToTensor(),\n        ])\n\n    class Custom_Dataset(torch.utils.data.Dataset):\n        def __init__(self,file_list,transformation_list = None):\n            self.file_list = file_list\n            self.transform = transformation_list\n\n        def __len__(self):\n            self.filelength = len(self.file_list)\n            return self.filelength\n\n        def __getitem__(self,idx):\n            img_path = self.file_list[idx]\n            img = Image.open(img_path)\n            img_transformed = self.transform(img)\n\n            label = img_path.split('/')[-1].split('.')[0]\n            if label == 'dog':\n                label=1\n            elif label == 'cat':\n                label=0\n\n            return img_transformed,label\n\n    train_dataset = Custom_Dataset(train_list, transformation_list)\n    val_dataset   = Custom_Dataset(val_list  , transformation_list)\n\n    train_loader  = torch.utils.data.DataLoader(dataset = train_dataset, batch_size=32, shuffle=True )\n    val_loader    = torch.utils.data.DataLoader(dataset = val_dataset, batch_size=32, shuffle=True)\n    \n    return train_dataset, val_dataset, train_loader, val_loader\n\n\ntraining_dataset, validation_dataset, training_dataloader, validation_dataloader = DOWNLOAD_DATASETS();\nassert next(iter(training_dataloader)) is not None\nassert next(iter(validation_dataloader)) is not None","metadata":{"_uuid":"96aafc3f-9cae-4abf-959b-f4d218b0cff9","_cell_guid":"8cff02e3-9730-4536-a8d7-06c0d6d206c5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-06-08T02:32:00.869726Z","iopub.execute_input":"2024-06-08T02:32:00.870400Z","iopub.status.idle":"2024-06-08T02:32:26.020938Z","shell.execute_reply.started":"2024-06-08T02:32:00.870365Z","shell.execute_reply":"2024-06-08T02:32:26.019898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```python\n# THIS TAKES at least 10 minutes to process vs Pytorch reading is a lot faster\nimport datasets\n\ndataset_from_hg            = datasets.load_dataset(\"microsoft/cats_vs_dogs\", split=\"train\", ignore_verifications= True )\ndef transform_datasets(examples):\n    examples[\"image_tensors\"] = []\n    for image in examples['image']:\n        transformed_image = transformations_group(image)\n        examples['image_tensors'].append(transformed_image)\n\n    return examples\n\ndataset_from_hg = dataset_from_hg   .map(transform_datasets  , batched= True)\n```","metadata":{}},{"cell_type":"markdown","source":"# 2. Simple Model Training Pipeline","metadata":{}},{"cell_type":"code","source":"lr      = 0.001 # learning_rate\nepochs  = 10 # How much to train a model\ndevice  = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef TRAIN_MODEL(model, training_dataloader, validation_dataloader):\n    \n    model.train(mode=True)\n    OPTIMIZER = torch.optim.SGD ( params= model.parameters(), lr= lr ) # Using torch.optimizer algorithm\n    metric    = torchmetrics.Accuracy(task=\"multiclass\", num_classes= 2 ).to(device)\n    \n    for epoch_no in range(epochs):\n        for batch_no, (image_tensors, labels) in enumerate(progress_bar := tqdm(training_dataloader)):\n            \n            x_actual, y_actual = image_tensors.to(device), labels.to(device)\n            \n            y_predicted_LOGITS = model.forward               (x_actual)\n            y_predicted_probs  = nn.functional.softmax       (y_predicted_LOGITS, dim= 1)\n            loss               = nn.functional.cross_entropy (y_predicted_LOGITS, y_actual.to(torch.int64))\n            \n            OPTIMIZER.zero_grad()\n            loss.backward()\n            # dError_dParameters    = torch.autograd.grad( outputs = ERROR_FUNC( y_predicted, y_actual ), inputs = model.parameters())\n            # Parameters of layer 1 are not dependent on any other parameters\n            # Parameters of layer 2 are dependent on layer 1 parameters\n            # Parameters of layer 3 are dependent on layer 2 parameters which are dependent on layer 1 parameters\n            # Finding complicated rate of change of such nested parameters is done automatically when we do loss.backward()\n            OPTIMIZER.step()\n            \"\"\"\n            for (name, weight), gradient in zip(model.named_parameters(), dError_dWeights):\n                weight = weight - gradient * LEARNING_RATE\n                print(f\"Parameters of layer: {name} have these many {torch.count_nonzero(gradient)} updates out of {torch.count(gradient)})\n            \"\"\"\n\n            loss_batch      = loss.item()\n            accuracy_batch  = metric(y_predicted_LOGITS, y_actual)\n            train_acc_epoch = metric.compute() # calculates average accuracy across epoch automatically\n\n            metrics_per_batch = {\n                \"loss_batch\": loss_batch,\n                \"accuracy_running_average\": train_acc_epoch,\n            }\n            progress_bar.set_description(f'batch_no = {batch_no},\\t loss_batch = {loss_batch:0.4f},\\t accuracy_avg = {train_acc_epoch:0.4f}')\n            \n        metric.reset()\n        loss_validation, accuracy_validation = EVALUATE_MODEL(model, validation_dataloader)\n        print(f'epoch_no = {epoch_no}, training_loss = {loss_batch:0.4f}, validation_loss = {loss_validation:0.4f},\\t training_accuracy = {accuracy_batch:0.4f}, validation_accuracy = {accuracy_validation:0.4f}')\n    \n    model.train(mode=False)\n\n\ndef EVALUATE_MODEL(model, validation_dataloader):\n    # EVALUATE MODEL AT END OF EVERY EPOCH\n    model.eval()\n    metric = torchmetrics.Accuracy(task=\"multiclass\", num_classes= 2 ).to(device)\n    with torch.no_grad():\n        for batch_no, (image_tensors, labels) in enumerate(validation_dataloader):\n            x_actual, y_actual = image_tensors.to(device), labels.to(device)\n\n            y_predicted_LOGITS = model.forward                 (x_actual)\n            loss               = nn.functional.cross_entropy   (y_predicted_LOGITS, y_actual.to(torch.int64)).item()\n            accuracy_batch     = metric                        (y_predicted_LOGITS, y_actual).item()\n\n        testing_accuracy_avg = metric.compute().item()\n    return loss, testing_accuracy_avg","metadata":{"_uuid":"ad119a64-e8f5-4201-8bce-f9f155b0f03e","_cell_guid":"e2dbe369-27ac-4420-ab3a-56f720b3825b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-06-08T02:34:56.549469Z","iopub.execute_input":"2024-06-08T02:34:56.549789Z","iopub.status.idle":"2024-06-08T02:34:56.590042Z","shell.execute_reply.started":"2024-06-08T02:34:56.549766Z","shell.execute_reply":"2024-06-08T02:34:56.589126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Model Architecture Experiments","metadata":{}},{"cell_type":"markdown","source":"## Experiment 1","metadata":{}},{"cell_type":"code","source":"feature_extractor = nn.Sequential(\n    # 224, 224\n    nn.Conv2d                           ( in_channels =  3, out_channels = 50, kernel_size = (3,3), padding=\"same\"), \n    nn.ReLU(), nn.MaxPool2d ((2,2), 2),                                           \n    # 112, 112\n    nn.Conv2d                           ( in_channels = 50, out_channels = 50, kernel_size = (3,3), padding=\"same\"),\n    nn.ReLU(), nn.MaxPool2d ((2,2), 2),\n    # 56, 56\n    nn.Conv2d                           ( in_channels = 50, out_channels = 50, kernel_size = (3,3), padding=\"same\"),\n    nn.ReLU(), nn.MaxPool2d ((2,2), 2),\n    # 28, 28\n    nn.Conv2d                           ( in_channels = 50, out_channels = 50, kernel_size = (3,3), padding=\"same\"),\n    nn.ReLU(), nn.MaxPool2d ((2,2), 2),\n    # 14, 14\n    nn.Conv2d                           ( in_channels = 50, out_channels = 512, kernel_size = (3,3), padding=\"same\"),\n    nn.ReLU(), nn.MaxPool2d ((2,2), 2),\n    # 7, 7\n)\n\ndecision_maker = nn.Sequential(\n    nn.Flatten(start_dim=1),\n    nn.Linear(in_features = 512*7*7 , out_features = 50), nn.ReLU(),\n    nn.Linear(in_features = 50      , out_features = 2)\n)\n\nmodel = nn.Sequential(\n  feature_extractor,\n  decision_maker\n)\nmodel = model.to(device)\n\ntest_example = torch.randn((1,3,224,224)).to(device)\nprint(feature_extractor(test_example).shape, model(test_example).shape)","metadata":{"_uuid":"286f8f56-e799-4852-9693-1717f7edca01","_cell_guid":"f1f536f5-f079-4c94-984d-7b56e01f79fd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-06-07T05:35:16.500511Z","iopub.execute_input":"2024-06-07T05:35:16.500950Z","iopub.status.idle":"2024-06-07T05:35:16.742635Z","shell.execute_reply.started":"2024-06-07T05:35:16.500915Z","shell.execute_reply":"2024-06-07T05:35:16.741187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_MODEL (model, training_dataloader, validation_dataloader)","metadata":{"_uuid":"2e1beb6d-2c67-409e-8761-0e5e171fd436","_cell_guid":"b7422417-423b-4e24-bfee-c666cc6de25a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-06-07T05:35:41.176655Z","iopub.execute_input":"2024-06-07T05:35:41.179788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Experiment 2","metadata":{}},{"cell_type":"code","source":"class Cnn(nn.Module):\n    def __init__(self):\n        super(Cnn,self).__init__()\n        \n        # Size Reduction 1/4th\n        self.layer1 = nn.Sequential(\n            nn.Conv2d(3,16, kernel_size=3, padding=0,stride=2),\n            nn.BatchNorm2d(16),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        # Size Reduction 1/4th\n        self.layer2 = nn.Sequential(\n            nn.Conv2d(16,32, kernel_size=3, padding=0, stride=2),\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n            )\n        \n        # Size Reduction 1/4th\n        self.layer3 = nn.Sequential(\n            nn.Conv2d(32,64, kernel_size=3, padding=0, stride=2),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        \n        \n        self.fc1     = nn.Linear(3*3*64,10)\n        self.dropout = nn.Dropout(0.5)\n        self.fc2     = nn.Linear(10,2)\n        self.relu    = nn.ReLU()\n        \n        \n    def forward(self,x):\n        out = self.layer1(x)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = out.view(out.size(0),-1)\n        out = self.relu(self.fc1(out))\n        out = self.fc2(out)\n        return out\n\nmodel = Cnn().to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-08T02:35:21.292021Z","iopub.execute_input":"2024-06-08T02:35:21.292494Z","iopub.status.idle":"2024-06-08T02:35:21.306818Z","shell.execute_reply.started":"2024-06-08T02:35:21.292460Z","shell.execute_reply":"2024-06-08T02:35:21.305903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_MODEL (model, training_dataloader, validation_dataloader)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torchinfo.summary(model, input_size= (1,3, 224, 224), verbose=1);","metadata":{"execution":{"iopub.status.busy":"2024-06-08T02:35:35.102387Z","iopub.execute_input":"2024-06-08T02:35:35.102749Z","iopub.status.idle":"2024-06-08T02:35:35.114486Z","shell.execute_reply.started":"2024-06-08T02:35:35.102721Z","shell.execute_reply":"2024-06-08T02:35:35.113499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image input & output size after Convolution\n$$\n\\begin{align*}\nimg &= (H,W,C)\\\\\nkernel &= (k,k,C)\\\\\noutput &= (reduced, reduced, 1)\\\\\n\\\\\nheight_{output} &= \\frac {(height_{input} + 2 \\cdot padding - (kernel - 1)  )}{stride} \\tag{1}\\\\\n\\end{align*}\n$$","metadata":{}},{"cell_type":"markdown","source":"## Receptive Field\n$$\n\\begin{align*}\nrf^{global} &= rf_{input} + (kernel - 1) \\cdot stride \\cdot s_{accum}\\\\\n\\\\\ns_{accum} &= s_{accum} \\cdot stride\\\\\n\\\\\ndepth &= \\frac {width} {(kernel -1) \\cdot s \\cdot s_{accum}}\n\\end{align*}\n$$","metadata":{}},{"cell_type":"markdown","source":"### Padding required for same output size\nStride = 1, padding for same height\n$$\n\\begin{align*}\ns &= 1 \\\\\npadding &= \\frac{( kernel - 1 )}{2}\\\\\n\\Delta rf &= kernel - 1\\\\\ndepth &= \\frac{width}{kernel - 1}\\\\\n\\end{align*}\n$$\n","metadata":{}},{"cell_type":"markdown","source":"## Going back to 28*28 size for Architecture experiementation","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\nclass Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        self.conv0 = nn.Conv2d(1,32, 3, padding=1)    # 28 -> 28 | 3\n\n        self.conv1 = nn.Conv2d(32, 64, 3, padding=1)  # 28 -> 28 | 5\n        self.conv2 = nn.Conv2d(64, 128, 3, padding=1) # 28 -> 28 | 7\n        self.pool1 = nn.MaxPool2d(2, 2)               # 28 > 14  | 14\n\n        self.conv3 = nn.Conv2d(128, 256, 3, padding=1)  # 14 -> 14 | 16\n        self.conv4 = nn.Conv2d(256, 512, 3, padding=1)  # 14 -> 14 | 18\n        self.pool2 = nn.MaxPool2d(2, 2)                 # 14 -> 7  | 36\n\n        self.conv5 = nn.Conv2d(512, 1024, 3)     # 7 -> 5 | 38\n        self.conv6 = nn.Conv2d(1024, 1024, 3)    # 5 -> 3 | 40\n\n        self.conv7 = nn.Conv2d(1024, 10, 3)     # 3 -> 1  | 42\n\n    def forward(self, x):\n        x = self.pool1(F.relu(self.conv2(F.relu(self.conv1(x)))))\n        x = self.pool2(F.relu(self.conv4(F.relu(self.conv3(x)))))\n        x = F.relu(self.conv6(F.relu(self.conv5(x))))\n        # x = F.relu(self.conv7(x))\n        x = self.conv7(x)\n        x = x.view(-1, 10) #1x1x10> 10\n        return F.log_softmax(x, dim=-1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model_1(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        # k:3, RF: 3 | Output: 28 -> 28 | Channels: (1 -> 32)\n        # Layer 1: Parameters = 32 * (1 * 3*3)\n        self.conv1 = nn.Conv2d(1, 32, 3, padding=1)\n\n        # k:3, RF: 5 | Output: 28 -> 28 | Channels: (32 -> 64)\n        # Layer 2: Parameters = 64 * (32 * 3*3)   \n        self.conv2 = nn.Conv2d(32, 64, 3, padding=1)\n\n        # k:2, RF: 10 | Output: 28 -> 14 | Channels: (64 -> 64)\n        # Layer 3: Parameters = 0\n        self.pool1 = nn.MaxPool2d(2, 2)\n\n        # k:3, RF: 12 | Output: 14 -> 14 | Channels: (64 -> 128)\n        # Layer 4: Parameters = 128 * (64 * 3*3)\n        self.conv3 = nn.Conv2d(64, 128, 3, padding=1)\n\n        # k:3, RF: 14 | Output: 14 -> 14 | Channels: (128 -> 256)\n        # Layer 5: Parameters = 256 * (128 * 3*3)\n        self.conv4 = nn.Conv2d(128, 256, 3, padding=1)\n\n        # k:2, RF: 28 | Output: 14 -> 7 | Channels: (256 -> 256)\n        # Layer 6: Parameters = 0\n        self.pool2 = nn.MaxPool2d(2, 2)\n\n        # k:3, RF: 30 | Output:  7 -> 5 | Channels: (256 -> 512)\n        # Layer 7: Parameters = 512 * (256 * 3*3)           \n        self.conv5 = nn.Conv2d(256, 512, 3)\n\n        # k:3, RF: 32 | Output:  5 -> 3 | Channels: (512 -> 1024)\n        # Layer 8: Parameters = 1024 * (512 * 3*3)        \n        self.conv6 = nn.Conv2d(512, 1024, 3)\n\n        # k:3, RF: 34 | Output:  3 -> 1 | Channels (1 -> 32)    \n        # Layer 9: Parameters = 1024 * (10 * 3*3)      \n        self.conv7 = nn.Conv2d(1024, 10, 3)             \n\n    def forward(self, x):\n        x = self.pool1(F.relu(self.conv2(F.relu(self.conv1(x)))))\n        x = self.pool2(F.relu(self.conv4(F.relu(self.conv3(x)))))\n        x = F.relu(self.conv6(F.relu(self.conv5(x))))\n        # x = F.relu(self.conv7(x))\n        x = self.conv7(x)\n\n        # 1*1*10 > (-1,10)\n        x = x.view(-1, 10)                              \n        # -1 means last dimention which is dim 1 in this case\n        # Two dimentions. dim 0 & dim 1\n        output = F.log_softmax(x, dim= 1) # OR F.log_softmax(x, dim=-1)\n        probs = F.softmax(x,dim=1)\n        \n        return output","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\nclass Model_2(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.conv0 = nn.Conv2d(1,32, 3, padding=1)    # 28 -> 28 | 3\n        \n        # Block 1\n        self.block1 = nn.ModuleDict({\n            \"conv1\": nn.Conv2d(32, 64, 3, padding=1),\n            \"relu1\" : nn.ReLU(),\n            \"conv2\": nn.Conv2d(64, 128, 3, padding=1),\n            \"relu2\" : nn.ReLU(),\n        })\n\n        # Maxpooling before or after 1x1 convolution?\n        self.transition1 = nn.Sequential(\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(128,32,1), # Squeeze\n        )\n\n        # Block 2\n        self.block2 = nn.ModuleDict({\n            \"conv1\": nn.Conv2d(32, 64, 3, padding=1),\n            \"relu1\" : nn.ReLU(),\n            \"conv2\": nn.Conv2d(64, 128, 3, padding=1),\n            \"relu2\" : nn.ReLU(),\n        })\n        \n        self.transition2 = nn.Sequential(\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(128,32,1), # Squeeze\n        )\n\n        # Block 3\n        self.block3 = nn.ModuleDict({\n            \"conv1\": nn.Conv2d(32, 64, 3, padding=1),\n            \"relu1\" : nn.ReLU(),\n            \"conv2\": nn.Conv2d(64, 128, 3, padding=1),\n            \"relu2\" : nn.ReLU(),\n        })\n\n        self.transition3 = nn.Sequential(\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(128,32,1), # Squeeze\n        )\n\n        # Block 4\n        self.block4 = nn.ModuleDict({\n            \"conv7\": nn.Conv2d(32, 10, 3),\n        })\n\n    def forward(self, x):\n        b1, b2, b3, b4 = self.block1, self.block2, self.block3, self.block4\n\n        x = self.conv0(x)\n\n        x = b1.relu2(b1.conv2(b1.relu1(b1.conv1(x))))\n        x = self.transition1(x)\n        \n        x = b2.relu2(b2.conv2(b2.relu1(b2.conv1(x))))\n        x = self.transition2(x)\n        \n        x = b3.relu2(b3.conv2(b3.relu1(b3.conv1(x))))\n        x = self.transition3(x)\n\n        x = b4.conv7(x)\n\n        # (-1 = dim 0, 10 = dim 1)\n        x = x.view(-1, 10)\n\n        output = F.log_softmax(x, dim=1)\n        probs = F.softmax(x, dim=1)\n        return output\n\nif __name__==\"__main__\":\n    # from torchsummary import summary\n\n    model_2 = Model_2()\n\n    for name in model_2.state_dict():\n        print(name)\n    \n    for module in model_2.modules():\n        print(module)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass Model_3(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.conv0 = nn.Conv2d(1,10, 3, padding=1)    # 28 -> 28 | 3\n        \n        # Block 1\n        self.block1 = nn.ModuleDict({\n            \"conv1\": nn.Conv2d(10, 10, 3, padding=1),\n            \"relu1\" : nn.ReLU(),\n            \"conv2\": nn.Conv2d(10, 20, 3, padding=1),\n            \"relu2\" : nn.ReLU(),\n        })\n\n        # Maxpooling before or after 1x1 convolution?\n        self.transition1 = nn.Sequential(\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(20,10,1), # Squeeze\n        )\n\n        # Block 2\n        self.block2 = nn.ModuleDict({\n            \"conv1\": nn.Conv2d(10, 10, 3, padding=1),\n            \"relu1\" : nn.ReLU(),\n            \"conv2\": nn.Conv2d(10, 20, 3, padding=1),\n            \"relu2\" : nn.ReLU(),\n        })\n        \n        self.transition2 = nn.Sequential(\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(20,10,1), # Squeeze\n        )\n\n        # Block 3\n        self.block3 = nn.ModuleDict({\n            \"conv1\": nn.Conv2d(10, 10, 3, padding=1),\n            \"relu1\" : nn.ReLU(),\n            \"conv2\": nn.Conv2d(10, 20, 3, padding=1),\n            \"relu2\" : nn.ReLU(),\n        })\n\n        self.transition3 = nn.Sequential(\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(20,10,1), # Squeeze\n        )\n\n        # Block 4\n        self.block4 = nn.ModuleDict({\n            \"conv7\": nn.Conv2d(10, 10, 3),\n        })\n\n    def forward(self, x):\n        b1, b2, b3, b4 = self.block1, self.block2, self.block3, self.block4\n\n        x = self.conv0(x)\n\n        x = b1.relu2(b1.conv2(b1.relu1(b1.conv1(x))))\n        x = self.transition1(x)\n        \n        x = b2.relu2(b2.conv2(b2.relu1(b2.conv1(x))))\n        x = self.transition2(x)\n        \n        x = b3.relu2(b3.conv2(b3.relu1(b3.conv1(x))))\n        x = self.transition3(x)\n\n        x = b4.conv7(x)\n\n        # (-1 = dim 0, 10 = dim 1)\n        x = x.view(-1, 10)\n\n        output = F.log_softmax(x, dim=1)\n        probs = F.softmax(x, dim=1)\n        return output\n\nif __name__==\"__main__\":\n    from torchsummary import summary\n\n    model_3 = Model_3()\n    summary(model_3,input_size=(1,28,28))\n\n    for name in model_3.state_dict():\n        print(name)\n    \n    # for module in model_3.modules():\n        # print(module)","metadata":{},"execution_count":null,"outputs":[]}]}