{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-11T16:19:56.749739Z","iopub.execute_input":"2024-03-11T16:19:56.750083Z","iopub.status.idle":"2024-03-11T16:19:57.124604Z","shell.execute_reply.started":"2024-03-11T16:19:56.750053Z","shell.execute_reply":"2024-03-11T16:19:57.123666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.datasets as dsets\ntrain_dataset =  dsets.CIFAR10(root='./data', train=True, download=True)\nvalidation_dataset = dsets.CIFAR10(root='./data', train=False, download=True)\nprint(len(train_dataset),len(validation_dataset))","metadata":{"execution":{"iopub.status.busy":"2024-03-11T16:19:57.126258Z","iopub.execute_input":"2024-03-11T16:19:57.126618Z","iopub.status.idle":"2024-03-11T16:20:07.376452Z","shell.execute_reply.started":"2024-03-11T16:19:57.126593Z","shell.execute_reply":"2024-03-11T16:20:07.375384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nX_labelled = []\nX1_unlabelled = []\n\nfor i in range(len(train_dataset)):\n    \n    if(i<5000):\n        X_labelled.append(train_dataset[i])\n    else:\n        X1_unlabelled.append(train_dataset[i])","metadata":{"execution":{"iopub.status.busy":"2024-03-11T16:20:07.377874Z","iopub.execute_input":"2024-03-11T16:20:07.378471Z","iopub.status.idle":"2024-03-11T16:20:09.743715Z","shell.execute_reply.started":"2024-03-11T16:20:07.378435Z","shell.execute_reply":"2024-03-11T16:20:09.742941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(X_labelled))\n","metadata":{"execution":{"iopub.status.busy":"2024-03-11T16:20:09.746006Z","iopub.execute_input":"2024-03-11T16:20:09.746446Z","iopub.status.idle":"2024-03-11T16:20:09.751357Z","shell.execute_reply.started":"2024-03-11T16:20:09.746414Z","shell.execute_reply":"2024-03-11T16:20:09.750392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(X1_unlabelled))","metadata":{"execution":{"iopub.status.busy":"2024-03-11T16:20:09.752442Z","iopub.execute_input":"2024-03-11T16:20:09.752761Z","iopub.status.idle":"2024-03-11T16:20:09.763141Z","shell.execute_reply.started":"2024-03-11T16:20:09.752732Z","shell.execute_reply":"2024-03-11T16:20:09.762324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nimport torchvision.transforms as transforms\nfrom torchvision import models\nimport torchvision\nfrom torch.utils.data import DataLoader, Dataset, Subset","metadata":{"execution":{"iopub.status.busy":"2024-03-11T16:20:09.764248Z","iopub.execute_input":"2024-03-11T16:20:09.765024Z","iopub.status.idle":"2024-03-11T16:20:09.773458Z","shell.execute_reply.started":"2024-03-11T16:20:09.764983Z","shell.execute_reply":"2024-03-11T16:20:09.772616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = [0.5, 0.5, 0.5]\nstd_dev = [0.5, 0.5, 0.5]\ntransform = transforms.Compose([transforms.Resize((32, 32)), transforms.PILToTensor(),transforms.Lambda(lambda x: add_gaussian_noise(x, mean=0.0, std=0.1))])\ndef add_gaussian_noise(tensor, mean=0.0, std=0.1):\n    tensor = tensor.float() \n    noise = torch.randn_like(tensor) * std + mean\n    return tensor + noise\nclass Loader_labelled(torch.utils.data.Dataset):\n    def __init__(self, images):\n        \n        self.images = images\n        \n        \n    def __getitem__(self, i):\n        \n        x = self.images[i][0]\n        x = transform(x)\n        #x = x.float() / 255.0 \n        \n        y1 = [0,0,0,0,0,0,0,0,0,0]\n        y = self.images[i][1]\n        y1[y] = 1\n        y1 = torch.tensor(y1)\n        return x, y1\n    \n    \n    def __len__(self):\n        return len(self.images)\n    \n\n","metadata":{"execution":{"iopub.status.busy":"2024-03-11T16:20:09.774530Z","iopub.execute_input":"2024-03-11T16:20:09.774780Z","iopub.status.idle":"2024-03-11T16:20:09.784932Z","shell.execute_reply.started":"2024-03-11T16:20:09.774759Z","shell.execute_reply":"2024-03-11T16:20:09.784031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Loader_unlabelled(torch.utils.data.Dataset):\n    \n    def __init__(self, images):\n        self.images = images\n        \n        \n    def __getitem__(self, i):\n        \n        x = self.images[i][0]\n        x=transform(x)\n        print(\"x\", x.float())\n        #x = x.float() / 255.0 \n        \n        return x\n        \n    def __len__(self):\n        return len(self.images)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T16:20:09.785860Z","iopub.execute_input":"2024-03-11T16:20:09.786107Z","iopub.status.idle":"2024-03-11T16:20:09.801005Z","shell.execute_reply.started":"2024-03-11T16:20:09.786086Z","shell.execute_reply":"2024-03-11T16:20:09.800256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labelled_dataset = Loader_labelled(X_labelled)\nunlabelled_dataset = Loader_unlabelled(X1_unlabelled)\n\nprint(len(labelled_dataset))\nprint(len(unlabelled_dataset))","metadata":{"execution":{"iopub.status.busy":"2024-03-11T16:20:09.803376Z","iopub.execute_input":"2024-03-11T16:20:09.803625Z","iopub.status.idle":"2024-03-11T16:20:09.810889Z","shell.execute_reply.started":"2024-03-11T16:20:09.803605Z","shell.execute_reply":"2024-03-11T16:20:09.810080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labeled_loader = torch.utils.data.DataLoader(labelled_dataset,shuffle = True)\ntrain_unlabeled_loader = torch.utils.data.DataLoader(unlabelled_dataset,shuffle = True)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:17:32.294878Z","iopub.execute_input":"2024-03-11T17:17:32.295581Z","iopub.status.idle":"2024-03-11T17:17:32.300095Z","shell.execute_reply.started":"2024-03-11T17:17:32.295552Z","shell.execute_reply":"2024-03-11T17:17:32.299162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Val=[]\nfor i in range(len(validation_dataset)):\n    if(i<1000):\n        Val.append(validation_dataset[i])\nlen(Val)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:17:34.185480Z","iopub.execute_input":"2024-03-11T17:17:34.185847Z","iopub.status.idle":"2024-03-11T17:17:34.235276Z","shell.execute_reply.started":"2024-03-11T17:17:34.185816Z","shell.execute_reply":"2024-03-11T17:17:34.234383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_dataset = Loader_labelled(Val)\n\nvalidation_loader = torch.utils.data.DataLoader(val_dataset, shuffle = True)\nprint(len(validation_loader))","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:17:35.597062Z","iopub.execute_input":"2024-03-11T17:17:35.597442Z","iopub.status.idle":"2024-03-11T17:17:35.602711Z","shell.execute_reply.started":"2024-03-11T17:17:35.597413Z","shell.execute_reply":"2024-03-11T17:17:35.601835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor i in train_labeled_loader:\n    print(i[0].shape,i[1])\n    count+=1\n    \n    if(count>5):\n        break\n","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:17:37.076859Z","iopub.execute_input":"2024-03-11T17:17:37.077243Z","iopub.status.idle":"2024-03-11T17:17:37.090957Z","shell.execute_reply.started":"2024-03-11T17:17:37.077207Z","shell.execute_reply":"2024-03-11T17:17:37.089727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ARCHITECTURE","metadata":{}},{"cell_type":"code","source":"from torchvision.models import resnet50\nstudent = resnet50()\nteacher = resnet50()\n\ndef exponential_moving_average(student, teacher, alpha):\n    for i,j in zip(student.parameters(), teacher.paramaters()):\n        j.data = alpha * (j.data) + (1 - alpha) * (i.data)\n        \ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nprint(device)\nstudent.to(device)\nteacher.to(device)\n    \n    \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:17:39.423717Z","iopub.execute_input":"2024-03-11T17:17:39.424437Z","iopub.status.idle":"2024-03-11T17:17:40.327276Z","shell.execute_reply.started":"2024-03-11T17:17:39.424406Z","shell.execute_reply":"2024-03-11T17:17:40.326348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F\ndef loss_function_labelled(outputs1, outputs2, labels):\n    labels = labels.view(-1)\n    labels = labels.long()\n    \n    #loss1 = nn.CrossEntropyLoss()(labels, outputs1)\n    loss1 = F.cross_entropy(outputs1, labels)\n    loss2 = torch.nn.MSELoss()(outputs1,outputs2)\n    \n    return torch.add(loss1, loss2)\n\ndef loss_function_unlabelled(outputs1, outputs, labels):\n    loss = torch.nn.MSELoss()(outputs1, outputs2)\n    return loss\n","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:17:41.041068Z","iopub.execute_input":"2024-03-11T17:17:41.041435Z","iopub.status.idle":"2024-03-11T17:17:41.047722Z","shell.execute_reply.started":"2024-03-11T17:17:41.041408Z","shell.execute_reply":"2024-03-11T17:17:41.046714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tqdm\n# optimizer = optim.SGD(student.parameters(), lr=0.01, momentum=0.9)\n# total_loss = 0\n\n# for i in range(50):\n    \n#     for data in tqdm.tqdm(train_labeled_loader, desc = f'Training Epoch: {i+1}/50', unit = 'batch'):\n#         #Training the labelled data\n#         inputs, labels = data\n#         labels = labels.long()\n#         inputs.to(device)\n#         labels.to(device)\n#         with torch.no_grad():\n#             teacher.eval()\n#             outputs2 = teacher(inputs.float())\n#         student.train()\n#         outputs1 = student(inputs.float())\n#         temp_loss = loss_function_labelled(outputs1, outputs2, labels)\n#         total_loss += temp_loss.item()\n#         optimizer.zerograd()\n#         temp_loss.backward()\n#         optimizer.step()\n#         exponential_moving_average(student, teacher, 0.99)\n        \n        \n#     for data in tqdm.tqdm(train_unlabeled_loader, desc = f'Training Epoch: {i+1}/50', unit = 'batch'):\n#         inputs = data\n#         inputs.to(device)\n#         with torch.no_grad:\n#             teacher.eval()\n#             outputs2 = teacher(inputs.float())\n#         student.train()\n#         outputs1 = student(inputs.float())\n#         temp_loss = loss_function_unlabelled(outputs1, outputs2, labels)\n#         total_loss+=temp_loss.item()\n#         optimizer.zero_grad()\n#         temp_loss.backward()\n#         optimizer.step()\n#         exponential_moving_average(student, teacher, 0.99)\n# #         print(\"Loss for Epoch:{} Loss:{}\".format(epoch+1,loss/50000))\nimport tqdm\n\noptimizer = optim.SGD(student.parameters(), lr=0.01, momentum=0.9)\n\ntotal_loss = 0\n\nfor i in range(50):\n    for data in tqdm.tqdm(train_labeled_loader, desc=f'Training Epoch: {i+1}/50', unit='batch'):\n        # Training the labeled data\n        inputs, labels = data\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n\n        with torch.no_grad():\n            teacher.eval()\n            outputs2 = teacher(inputs)\n\n        student.train()\n        outputs1 = student(inputs)\n        temp_loss = loss_function_labelled(outputs1, outputs2, labels)\n        total_loss += temp_loss.item()\n\n        optimizer.zero_grad()\n        temp_loss.backward()\n        optimizer.step()\n        exponential_moving_average(student, teacher, 0.99)\n\n    for data in tqdm.tqdm(train_unlabeled_loader, desc=f'Training Epoch: {i+1}/50', unit='batch'):\n        inputs = data\n        inputs = inputs.to(device)\n\n        with torch.no_grad():\n            teacher.eval()\n            outputs2 = teacher(inputs)\n\n        student.train()\n        outputs1 = student(inputs)\n        temp_loss = loss_function_unlabelled(outputs1, outputs2)\n        total_loss += temp_loss.item()\n\n        optimizer.zero_grad()\n        temp_loss.backward()\n        optimizer.step()\n        exponential_moving_average(student, teacher, 0.99)\n\n    print(\"Loss for Epoch:{} Loss:{}\".format(i+1, total_loss/50000))","metadata":{"execution":{"iopub.status.busy":"2024-03-11T17:17:42.415712Z","iopub.execute_input":"2024-03-11T17:17:42.416694Z","iopub.status.idle":"2024-03-11T17:17:42.952580Z","shell.execute_reply.started":"2024-03-11T17:17:42.416661Z","shell.execute_reply":"2024-03-11T17:17:42.951178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n","metadata":{},"execution_count":null,"outputs":[]}]}