{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nfrom tensorflow.keras.datasets import cifar10\n(X_train, Y_train), (X_test, Y_test) = cifar10.load_data()\n\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(10, 5))\nfor i in range(5):\n    plt.subplot(1, 5, i + 1)\n    plt.imshow(X_train[i])\n    plt.title(f\"Class: {Y_train[i][0]}\")\n    plt.axis('off')\n\nplt.show()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-10T16:39:06.615334Z","iopub.execute_input":"2024-03-10T16:39:06.615746Z","iopub.status.idle":"2024-03-10T16:39:07.621679Z","shell.execute_reply.started":"2024-03-10T16:39:06.615720Z","shell.execute_reply":"2024-03-10T16:39:07.620740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport numpy as np\nfrom PIL import Image \n\nBATCH_SIZE = 16\n\ndef add_gaussian_noise(img, sigma=0.15):\n    noise = torch.randn_like(img) * sigma\n    noisy_img = img + noise\n    return noisy_img.clamp(0, 1)\n\ntranslate_range = (2 / 32, 2 / 32)\n\ndata_transform = transforms.Compose([\n    transforms.Lambda(lambda x: Image.fromarray(x)),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomAffine(degrees=0, translate=translate_range),\n    \n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) \n])\n\nclass CustomDataset(Dataset):\n    def __init__(self, data, labels, transform=None, labeled_samples=1000):\n        self.data = data\n        self.labels = np.concatenate((labels[:labeled_samples], -1 * np.ones((len(data) - labeled_samples, 1), dtype=np.int64)))\n        self.transform = transform\n        self.labeled_mask = np.zeros(len(self.data), dtype=bool)\n        self.labeled_mask[:labeled_samples] = True\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img, label = self.data[idx], self.labels[idx]\n\n        if self.transform:\n            img = self.transform(img)\n\n        if self.labeled_mask[idx]:\n            return img, label\n        else:\n            return img, -1\n\naugmented_dataset = CustomDataset(X_train, Y_train, transform=data_transform, labeled_samples=1000)","metadata":{"execution":{"iopub.status.busy":"2024-03-10T16:39:07.623483Z","iopub.execute_input":"2024-03-10T16:39:07.624207Z","iopub.status.idle":"2024-03-10T16:39:07.635958Z","shell.execute_reply.started":"2024-03-10T16:39:07.624172Z","shell.execute_reply":"2024-03-10T16:39:07.634914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LabeledDataset(Dataset):\n    def __init__(self, original_dataset):\n        self.original_dataset = original_dataset\n        self.indices = [idx for idx in range(original_dataset.__len__()) if (original_dataset.__getitem__(idx)[1] != -1)]\n\n    def __len__(self):\n        return len(self.indices)\n\n    def __getitem__(self, idx):\n        original_idx = self.indices[idx]\n        return self.original_dataset[original_idx]\n\nlabeled_dataset = LabeledDataset(augmented_dataset)\n\n\nfrom torch.utils.data import Subset, random_split\npretrain_size = int(0.8 * 1000)\npreval_size = 1000 - pretrain_size\npretrain_dataset, preval_dataset = random_split(labeled_dataset,[pretrain_size,preval_size])\n\npretrain_loader = DataLoader(pretrain_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\npreval_loader = DataLoader(preval_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2024-03-10T16:39:07.636888Z","iopub.execute_input":"2024-03-10T16:39:07.637180Z","iopub.status.idle":"2024-03-10T16:39:26.892991Z","shell.execute_reply.started":"2024-03-10T16:39:07.637157Z","shell.execute_reply":"2024-03-10T16:39:26.891958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\n\nclass pre(nn.Module):\n    def __init__(self, num_classes=10, dropout_p=0.5):\n        super(pre, self).__init__()\n        self.conv1 = nn.Conv2d(3, 128, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(128, 128, kernel_size=3, padding=1)\n        self.conv3 = nn.Conv2d(128, 128, kernel_size=3, padding=1)\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n\n        self.conv4 = nn.Conv2d(128, 256, kernel_size=3, padding=1)\n        self.conv5 = nn.Conv2d(256, 256, kernel_size=3, padding=1)\n        self.conv6 = nn.Conv2d(256, 256, kernel_size=3, padding=1)\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n\n        self.conv7 = nn.Conv2d(256, 512, kernel_size=3, padding=0)\n        self.conv8 = nn.Conv2d(512, 256, kernel_size=1, padding=0)\n        self.conv9 = nn.Conv2d(256, 128, kernel_size=1, padding=0)\n        self.avg_pool = nn.AvgPool2d(kernel_size=6)\n        \n        self.fc = nn.Linear(128, num_classes)\n\n    def forward(self, x):\n        x = F.leaky_relu(self.conv1(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv2(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv3(x), negative_slope=0.1)\n\n        x = self.pool1(x)\n\n        x = F.leaky_relu(self.conv4(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv5(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv6(x), negative_slope=0.1)\n        \n        x = self.pool2(x)\n\n        x = F.leaky_relu(self.conv7(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv8(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv9(x), negative_slope=0.1)\n        x = self.avg_pool(x)\n        x = x.view(x.size(0), -1)\n        x = torch.transpose(x, 0, 1)\n        x = self.fc(x)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-03-10T16:39:26.895760Z","iopub.execute_input":"2024-03-10T16:39:26.896163Z","iopub.status.idle":"2024-03-10T16:39:26.910063Z","shell.execute_reply.started":"2024-03-10T16:39:26.896128Z","shell.execute_reply":"2024-03-10T16:39:26.909146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm \nimport torch.optim as optim\nimport torch.nn.functional as F\n\nmodel = pre()\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n    running_loss = 0.0\n    for inputs,labels in pretrain_dataset:\n        optimizer.zero_grad()\n        \n        outputs = model(inputs)\n        loss = criterion(outputs, torch.from_numpy(labels).long())\n\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n    print(f'Epoch {epoch + 1},Loss: {running_loss / len(pretrain_dataset)}')\n\nmodel.eval()\ncorrect = 0\ntotal = 0\nwith torch.no_grad():\n    for inputs, labels in preval_dataset:\n        outputs = model(inputs)\n        _, predicted = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n\nprint(f'Accuracy: {100 * correct / total}%')","metadata":{"execution":{"iopub.status.busy":"2024-03-10T16:39:26.911298Z","iopub.execute_input":"2024-03-10T16:39:26.911632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Subset, random_split\n\ntotal_size = len(X_train)\n\ntrain_size = int(0.8 * total_size)\nval_size = total_size - train_size\n\ntrain_dataset, val_dataset = random_split(\n    augmented_dataset, [train_size, val_size])\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass StudentModel(nn.Module):\n    def __init__(self, num_classes=10, dropout_p=0.5, pretrained_weights=None):\n        super(StudentModel, self).__init__()\n        self.conv1 = nn.Conv2d(3, 128, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(128, 128, kernel_size=3, padding=1)\n        self.conv3 = nn.Conv2d(128, 128, kernel_size=3, padding=1)\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.dropout1 = nn.Dropout2d(p=dropout_p)\n\n        self.conv4 = nn.Conv2d(128, 256, kernel_size=3, padding=1)\n        self.conv5 = nn.Conv2d(256, 256, kernel_size=3, padding=1)\n        self.conv6 = nn.Conv2d(256, 256, kernel_size=3, padding=1)\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.dropout2 = nn.Dropout2d(p=dropout_p)\n\n        self.conv7 = nn.Conv2d(256, 512, kernel_size=3, padding=0)\n        self.conv8 = nn.Conv2d(512, 256, kernel_size=1, padding=1)\n        self.conv9 = nn.Conv2d(256, 128, kernel_size=1, padding=1)\n        self.avgpool = nn.AvgPool2d(kernel_size=6, stride=1)\n\n        self.fc = nn.Linear(128, num_classes)\n        self.pretrained_weights = pretrained_weights\n        if self.pretrained_weights is not None:\n            self.load_state_dict(self.pretrained_weights)\n\n    def forward(self, x):\n        x = F.leaky_relu(self.conv1(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv2(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv3(x), negative_slope=0.1)\n        x = self.pool1(x)\n        x = self.dropout1(x)\n\n        x = F.leaky_relu(self.conv4(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv5(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv6(x), negative_slope=0.1)\n        x = self.pool2(x)\n        x = self.dropout2(x)\n\n        x = F.leaky_relu(self.conv7(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv8(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv9(x), negative_slope=0.1)\n        x = self.avgpool(x)\n        \n        x = x.view(x.size(0), -1)\n        x = x.view(x.size(0), 128)\n        \n        x = self.fc(x)\n        \n        return x\n\nclass TeacherModel(nn.Module):\n    def __init__(self, num_classes=10, dropout_p=0.5):\n        super(TeacherModel, self).__init__()\n        self.conv1 = nn.Conv2d(3, 128, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(128, 128, kernel_size=3, padding=1)\n        self.conv3 = nn.Conv2d(128, 128, kernel_size=3, padding=1)\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.dropout1 = nn.Dropout2d(p=dropout_p)\n\n        self.conv4 = nn.Conv2d(128, 256, kernel_size=3, padding=1)\n        self.conv5 = nn.Conv2d(256, 256, kernel_size=3, padding=1)\n        self.conv6 = nn.Conv2d(256, 256, kernel_size=3, padding=1)\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.dropout2 = nn.Dropout2d(p=dropout_p)\n\n        self.conv7 = nn.Conv2d(256, 512, kernel_size=3, padding=0)\n        self.conv8 = nn.Conv2d(512, 256, kernel_size=1, padding=1)\n        self.conv9 = nn.Conv2d(256, 128, kernel_size=1, padding=1)\n        self.avgpool = nn.AvgPool2d(kernel_size=6, stride=1)\n\n        self.fc = nn.Linear(128, num_classes)\n\n    def forward(self, x):\n        x = F.leaky_relu(self.conv1(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv2(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv3(x), negative_slope=0.1)\n        x = self.pool1(x)\n        x = self.dropout1(x)\n\n        x = F.leaky_relu(self.conv4(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv5(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv6(x), negative_slope=0.1)\n        x = self.pool2(x)\n        x = self.dropout2(x)\n\n        x = F.leaky_relu(self.conv7(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv8(x), negative_slope=0.1)\n        x = F.leaky_relu(self.conv9(x), negative_slope=0.1)\n        x = self.avgpool(x)\n\n        x = x.view(x.size(0), -1)\n\n        x = self.fc(x)\n        \n        return x\n\nclass MeanTeacher:\n    def __init__(self, num_classes=10, dropout_p=0.5, alpha=0.999, pretrained_weights=None):\n        self.student_model = StudentModel(num_classes, dropout_p, pretrained_weights)\n        self.teacher_model = TeacherModel(num_classes, dropout_p)\n        self.alpha = alpha\n\n        for teacher_param, student_param in zip(self.teacher_model.parameters(), self.student_model.parameters()):\n            teacher_param.data.copy_(student_param.data)\n\n    def update_teacher_model(self):\n        for teacher_param, student_param in zip(self.teacher_model.parameters(), self.student_model.parameters()):\n            teacher_param.data = self.alpha * teacher_param.data + (1 - self.alpha) * student_param.data\n\n    def train(self, data_loader, optimizer, criterion, num_epochs):\n        for epoch in range(num_epochs):\n            for inputs, targets in data_loader:\n                student_outputs = self.student_model(inputs)\n                loss = criterion(student_outputs, targets)\n\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                self.update_teacher_model()\n\n    def evaluate(self, data_loader):\n\n        pass\n\n\nmean_teacher = MeanTeacher()\n\nmean_teacher.train(train_loader, optimizer, criterion, num_epochs)\n\nmean_teacher.evaluate(test_loader)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}