{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-12T10:06:59.294248Z","iopub.execute_input":"2024-03-12T10:06:59.294615Z","iopub.status.idle":"2024-03-12T10:06:59.305998Z","shell.execute_reply.started":"2024-03-12T10:06:59.294582Z","shell.execute_reply":"2024-03-12T10:06:59.305034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:06:59.307690Z","iopub.execute_input":"2024-03-12T10:06:59.307979Z","iopub.status.idle":"2024-03-12T10:06:59.314289Z","shell.execute_reply.started":"2024-03-12T10:06:59.307956Z","shell.execute_reply":"2024-03-12T10:06:59.313426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport py7zr\nwith py7zr.SevenZipFile('/kaggle/input/cifar-10/train.7z') as archive:\n    for file in archive.getnames():\n        archive.extract(file)\n        print(file)\n'''","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:06:59.317898Z","iopub.execute_input":"2024-03-12T10:06:59.318720Z","iopub.status.idle":"2024-03-12T10:06:59.324792Z","shell.execute_reply.started":"2024-03-12T10:06:59.318686Z","shell.execute_reply":"2024-03-12T10:06:59.324022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''import pandas as pd\ndf = pd.read_csv('/kaggle/input/cifar-10/trainLabels.csv')\nprint(df)'''","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:06:59.326197Z","iopub.execute_input":"2024-03-12T10:06:59.326516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''import numpy as np\narray = df.to_numpy()\nprint(array)'''","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:06:59.337528Z","iopub.execute_input":"2024-03-12T10:06:59.337946Z","iopub.status.idle":"2024-03-12T10:06:59.345226Z","shell.execute_reply.started":"2024-03-12T10:06:59.337914Z","shell.execute_reply":"2024-03-12T10:06:59.344339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ('airplane', \n           'automobile', \n           'bird',\n           'cat',\n           'deer',\n           'dog', \n           'frog', \n           'horse', \n           'ship',\n           'truck')\nlen(classes)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:06:59.346287Z","iopub.execute_input":"2024-03-12T10:06:59.346567Z","iopub.status.idle":"2024-03-12T10:06:59.355766Z","shell.execute_reply.started":"2024-03-12T10:06:59.346544Z","shell.execute_reply":"2024-03-12T10:06:59.355000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.datasets as dsets\ntrain_dataset =  dsets.CIFAR10(root='./data', train=True, download=True)\nvalidation_dataset = dsets.CIFAR10(root='./data', train=False, download=True)\nprint(len(train_dataset),len(validation_dataset))","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:06:59.357077Z","iopub.execute_input":"2024-03-12T10:06:59.357453Z","iopub.status.idle":"2024-03-12T10:07:00.940416Z","shell.execute_reply.started":"2024-03-12T10:06:59.357428Z","shell.execute_reply":"2024-03-12T10:07:00.939379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(train_dataset[0][0])","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:07:00.941621Z","iopub.execute_input":"2024-03-12T10:07:00.941931Z","iopub.status.idle":"2024-03-12T10:07:00.948751Z","shell.execute_reply.started":"2024-03-12T10:07:00.941902Z","shell.execute_reply":"2024-03-12T10:07:00.947894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=[]\nX1=[]\nfor i in range(len(train_dataset)):\n    if(i<1000):\n        X.append(train_dataset[i])\n    else:\n        if(i<10000):\n            X1.append(train_dataset[i])\n        else:\n            break","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:07:00.951259Z","iopub.execute_input":"2024-03-12T10:07:00.951882Z","iopub.status.idle":"2024-03-12T10:07:01.365029Z","shell.execute_reply.started":"2024-03-12T10:07:00.951845Z","shell.execute_reply":"2024-03-12T10:07:01.364163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"V=[]\nfor i in range(500):\n    V.append(validation_dataset[i])","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:07:09.833095Z","iopub.execute_input":"2024-03-12T10:07:09.833442Z","iopub.status.idle":"2024-03-12T10:07:09.867369Z","shell.execute_reply.started":"2024-03-12T10:07:09.833417Z","shell.execute_reply":"2024-03-12T10:07:09.866621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as transforms\nimport torch\ntransform = transforms.Compose([ \n    transforms.PILToTensor() \n])\nclass cifar_10_(torch.utils.data.Dataset):\n    def __init__(self,images):\n        self.images = images\n        \n    def __getitem__(self,i):\n        x = transform(self.images[i][0])\n        x = x/255 \n        y = self.images[i][1]\n        y1 = [0,0,0,0,0,0,0,0,0,0]\n        y1[y] += 1\n        y1 = torch.tensor(y1)\n        return x,y1\n    def __len__(self):\n        return len(self.images)\n\nclass cifar_10_unlabled(torch.utils.data.Dataset):\n    def __init__(self,images):\n        self.images = images\n        \n    def __getitem__(self,i):\n        x = transform(self.images[i][0])/255\n        return x\n    def __len__(self):\n        return len(self.images)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:02.269119Z","iopub.execute_input":"2024-03-12T10:18:02.270046Z","iopub.status.idle":"2024-03-12T10:18:02.279011Z","shell.execute_reply.started":"2024-03-12T10:18:02.270010Z","shell.execute_reply":"2024-03-12T10:18:02.278017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labled_dataset = cifar_10_(X)\nunlabled_dataset = cifar_10_unlabled(X1)\nprint(len(labled_dataset),len(unlabled_dataset))","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:05.745411Z","iopub.execute_input":"2024-03-12T10:18:05.745859Z","iopub.status.idle":"2024-03-12T10:18:05.751721Z","shell.execute_reply.started":"2024-03-12T10:18:05.745821Z","shell.execute_reply":"2024-03-12T10:18:05.750639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_val = cifar_10_(V)\nprint(len(dataset_val))","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:07:20.030205Z","iopub.execute_input":"2024-03-12T10:07:20.030549Z","iopub.status.idle":"2024-03-12T10:07:20.035556Z","shell.execute_reply.started":"2024-03-12T10:07:20.030523Z","shell.execute_reply":"2024-03-12T10:07:20.034641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_l_loader = torch.utils.data.DataLoader(labled_dataset,shuffle=True)\ntrain_un_loader = torch.utils.data.DataLoader(unlabled_dataset,shuffle=True)\nval_loader = torch.utils.data.DataLoader(dataset_val,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:10.833531Z","iopub.execute_input":"2024-03-12T10:18:10.834377Z","iopub.status.idle":"2024-03-12T10:18:10.840432Z","shell.execute_reply.started":"2024-03-12T10:18:10.834343Z","shell.execute_reply":"2024-03-12T10:18:10.839505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_l_loader:\n    print(i[0].shape,i[1])\n    break","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:14.466689Z","iopub.execute_input":"2024-03-12T10:18:14.467064Z","iopub.status.idle":"2024-03-12T10:18:14.475052Z","shell.execute_reply.started":"2024-03-12T10:18:14.467036Z","shell.execute_reply":"2024-03-12T10:18:14.473955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nclass Cifar10CnnModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.network = nn.Sequential(\n            nn.Conv2d(3, 32, kernel_size=3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2, 2), # output: 64 x 16 x 16\n\n            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2, 2), # output: 128 x 8 x 8\n\n            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2, 2), # output: 256 x 4 x 4\n\n            nn.Flatten(), \n            nn.Linear(256*4*4, 1024),\n            #nn.Dropout(p),\n            nn.ReLU(),\n            nn.Linear(1024, 512),\n            #nn.Dropout(p),\n            nn.ReLU(),\n            nn.Linear(512, 10),\n            nn.Softmax(dim =1)\n        )\n        \n    def forward(self,x):\n        return self.network(x)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:18.412607Z","iopub.execute_input":"2024-03-12T10:18:18.413621Z","iopub.status.idle":"2024-03-12T10:18:18.423314Z","shell.execute_reply.started":"2024-03-12T10:18:18.413578Z","shell.execute_reply":"2024-03-12T10:18:18.422321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"student = Cifar10CnnModel()\nteacher = Cifar10CnnModel()\nstudent","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:21.240944Z","iopub.execute_input":"2024-03-12T10:18:21.241607Z","iopub.status.idle":"2024-03-12T10:18:21.343125Z","shell.execute_reply.started":"2024-03-12T10:18:21.241576Z","shell.execute_reply":"2024-03-12T10:18:21.342276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda:0\"\nstudent.to(device)\nteacher.to(device)\nstudent\nteacher","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:23.465941Z","iopub.execute_input":"2024-03-12T10:18:23.466553Z","iopub.status.idle":"2024-03-12T10:18:23.486434Z","shell.execute_reply.started":"2024-03-12T10:18:23.466523Z","shell.execute_reply":"2024-03-12T10:18:23.485628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_l_loader:\n    j = i[0].to(device)\n    print(i[0])\n    print(student(j.float()))\n    break","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:26.896517Z","iopub.execute_input":"2024-03-12T10:18:26.896921Z","iopub.status.idle":"2024-03-12T10:18:26.915931Z","shell.execute_reply.started":"2024-03-12T10:18:26.896892Z","shell.execute_reply":"2024-03-12T10:18:26.915047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_funct_label(labels,outputs,outputs1):\n    x = torch.nn.CrossEntropyLoss()(outputs,labels.float())\n    x1 = torch.nn.MSELoss()(outputs,outputs1)\n    return x\n\ndef loss_funct_unlabel(outputs,outputs1):\n    #x = torch.nn.CrossEntropyLoss()(labels,outputs)\n    x1 = torch.nn.MSELoss()(outputs,outputs1)\n    return x1\n    ","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:47.285442Z","iopub.execute_input":"2024-03-12T10:18:47.286043Z","iopub.status.idle":"2024-03-12T10:18:47.294227Z","shell.execute_reply.started":"2024-03-12T10:18:47.286013Z","shell.execute_reply":"2024-03-12T10:18:47.293331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_teacher_params(student, teacher, alpha, global_step):\n    alpha = min(1 - 1 / (global_step + 1), alpha)\n    for ema_param, param in zip(teacher.parameters(), student.parameters()):\n        ema_param.data.mul_(alpha).add_(1 - alpha, param.data)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:18:49.122767Z","iopub.execute_input":"2024-03-12T10:18:49.123677Z","iopub.status.idle":"2024-03-12T10:18:49.128831Z","shell.execute_reply.started":"2024-03-12T10:18:49.123644Z","shell.execute_reply":"2024-03-12T10:18:49.127956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\n\noptimizer = torch.optim.Adam(student.parameters(), lr=0.01)\nglobal_step = 0\nloss_lowest = 1\nnum = 0\nfor epoch in range(100):\n    loss = 0\n    g = global_step\n    for data in tqdm.tqdm(train_l_loader,desc=f'Training Labled Epoch {epoch + 1}/100', unit='batch'):\n        tea_pred = []\n        g += 1\n        inputs,labels = data\n        labels = labels.long()\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n        #teacher.eval()\n        outputs1 = teacher(inputs.float())\n        student.train()\n        outputs = student(inputs.float())\n        optimizer.zero_grad()\n        #l1 = loss_funct_label(labels,outputs,outputs1)\n        l1=torch.nn.functional.binary_cross_entropy(outputs,labels.float())\n        l1.backward()\n        optimizer.step()\n        loss += l1.item()\n    update_teacher_params(student,teacher,0.5,g)\n    g = global_step\n    loss1 = loss\n    print(\"Loss for labled = \",loss/len(train_l_loader))\n    for data in tqdm.tqdm(train_un_loader,desc=f'Training Unlabled Epoch {epoch + 1}/100', unit='batch'):\n        tea_pred = []\n        g += 1\n        inputs = data\n        inputs = inputs.to(device)\n        teacher.eval()\n        outputs1 = teacher(inputs.float())\n        student.train()\n        outputs = student(inputs.float())\n        l1 = loss_funct_unlabel(outputs,outputs1)\n        loss += l1.item()\n        optimizer.zero_grad()\n        l1.backward()\n        optimizer.step()\n    update_teacher_params(student,teacher,0.5,g)\n    global_step = g\n    print(\"Loss for Epoch:{} Loss:{}\".format(epoch+1,(loss-loss1)/(len(train_l_loader)+len(train_un_loader))))\n    student.eval()\n    loss = 0\n    for data in tqdm.tqdm(val_loader,desc=f'Validation Epoch {epoch + 1}/100', unit='batch'):\n        with torch.no_grad():\n            inputs,labels = data\n            labels = labels.long()\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            outputs = student(inputs.float())\n            l = torch.nn.CrossEntropyLoss()(outputs,labels.float())\n            loss += l\n    print(\"Validation Loss for Epoch:{} Loss:{}\".format(epoch+1,loss/len(val_loader)))\n    if(loss/len(val_loader) < loss_lowest):\n        loss_lowest = loss_per_epoch/len(val_loader)\n        torch.save(student, 'best_student.pth')\n        num = 0\n        print('saved')\n    else:\n        num = num + 1\n        if(num > 10):\n            print(\"Breaking\")\n            break\n    ","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:19:07.470363Z","iopub.execute_input":"2024-03-12T10:19:07.470734Z","iopub.status.idle":"2024-03-12T10:21:54.709424Z","shell.execute_reply.started":"2024-03-12T10:19:07.470703Z","shell.execute_reply":"2024-03-12T10:21:54.708213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\n\noptimizer = torch.optim.Adam(student.parameters(), lr=0.1)\nglobal_step = 0\nloss_lowest = 1\nnum = 0\nfor epoch in range(100):\n    loss = 0\n    g = global_step\n    for data in tqdm.tqdm(train_l_loader,desc=f'Training Labled Epoch {epoch + 1}/100', unit='batch'):\n        tea_pred = []\n        g += 1\n        inputs,labels = data\n        labels = labels.long()\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n        #teacher.eval()\n        outputs1 = teacher(inputs.float())\n        student.train()\n        outputs = student(inputs.float())\n        optimizer.zero_grad()\n        #l1 = loss_funct_label(labels,outputs,outputs1)\n        l1=torch.nn.functional.binary_cross_entropy()(outputs,labels)\n        l1.backward()\n        optimizer.step()\n        loss += l1.item()\n    update_teacher_params(student,teacher,0.5,g)\n    g = global_step\n    loss1 = loss\n    print(\"Loss for labled = \",loss/len(train_l_loader))","metadata":{"execution":{"iopub.status.busy":"2024-03-12T10:24:53.065461Z","iopub.execute_input":"2024-03-12T10:24:53.066175Z"},"trusted":true},"execution_count":null,"outputs":[]}]}