{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision import datasets\nfrom torch.utils.data import DataLoader\nfrom torch.optim import Adam\nfrom torchvision.transforms import ToTensor\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-28T04:13:46.027329Z","iopub.execute_input":"2024-03-28T04:13:46.027700Z","iopub.status.idle":"2024-03-28T04:13:46.033300Z","shell.execute_reply.started":"2024-03-28T04:13:46.027673Z","shell.execute_reply":"2024-03-28T04:13:46.032179Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if torch.cuda.is_available():\n    device=torch.device(type='cuda',index=0)\nelse:\n    device=torch.device(type='cpu',index=0)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:13:46.034957Z","iopub.execute_input":"2024-03-28T04:13:46.035293Z","iopub.status.idle":"2024-03-28T04:13:46.044325Z","shell.execute_reply.started":"2024-03-28T04:13:46.035270Z","shell.execute_reply":"2024-03-28T04:13:46.043521Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset=datasets.MNIST(root=\"/kaggle/temp/\",\n                            train=True,\n                            download=True,\n                            transform=ToTensor())\n\neval_dataset=datasets.MNIST(root=\"/kaggle/temp/\",\n                            train=False,\n                            download=True,\n                            transform=ToTensor())\n\nbatch_size=64\n\ntrain_dataloader=DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\neval_dataloader=DataLoader(dataset=eval_dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:13:46.045285Z","iopub.execute_input":"2024-03-28T04:13:46.045562Z","iopub.status.idle":"2024-03-28T04:13:46.137409Z","shell.execute_reply.started":"2024-03-28T04:13:46.045539Z","shell.execute_reply":"2024-03-28T04:13:46.136409Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Encoder1(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.lkrelu=nn.LeakyReLU(0.1)\n        self.l1=nn.Linear(in_features=784,out_features=512)\n        self.l2=nn.Linear(in_features=512,out_features=256)\n    \n    def forward(self,x):\n        net1=self.l1(x)\n        out1=self.lkrelu(net1)\n        net2=self.l2(out1)\n        out2=self.lkrelu(net2)\n        return out2\n    \nclass Decoder1(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.lkrelu=nn.LeakyReLU(0.1)\n        self.sig=nn.Sigmoid()\n        self.l1=nn.Linear(in_features=256,out_features=512)\n        self.l2=nn.Linear(in_features=512,out_features=784)\n    \n    def forward(self,x):\n        net1=self.l1(x)\n        out1=self.lkrelu(net1)\n        net2=self.l2(out1)\n        out2=self.sig(net2)\n        return out2\n\nclass Encoder2(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.lkrelu=nn.LeakyReLU(0.1)\n        self.l1=nn.Linear(in_features=256,out_features=100)\n            \n    def forward(self,x):\n        net1=self.l1(x)\n        out1=self.lkrelu(net1)\n        return out1\n    \nclass Decoder2(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.lkrelu=nn.LeakyReLU(0.1)\n        self.l1=nn.Linear(in_features=100,out_features=256)\n            \n    def forward(self,x):\n        net1=self.l1(x)\n        out1=self.lkrelu(net1)\n        return out1\n\n\nclass Classifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.out=nn.Linear(in_features=100,out_features=10)\n    \n    def forward(self,x):\n        out=self.out(x)\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:13:46.138961Z","iopub.execute_input":"2024-03-28T04:13:46.139281Z","iopub.status.idle":"2024-03-28T04:13:46.152549Z","shell.execute_reply.started":"2024-03-28T04:13:46.139257Z","shell.execute_reply":"2024-03-28T04:13:46.151557Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lossmse_fn=nn.MSELoss()\nlossentropy_fn=nn.CrossEntropyLoss()\nlr=0.001\n\ne1=Encoder1().to(device)\nd1=Decoder1().to(device)\ne2=Encoder2().to(device)\nd2=Decoder2().to(device)\nclf=Classifier().to(device)\n\nopte1=Adam(params=e1.parameters(),lr=lr)\noptd1=Adam(params=d1.parameters(),lr=lr)\nopte2=Adam(params=e2.parameters(),lr=lr)\noptd2=Adam(params=d2.parameters(),lr=lr)\noptclf=Adam(params=clf.parameters(),lr=lr)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:13:46.154573Z","iopub.execute_input":"2024-03-28T04:13:46.154844Z","iopub.status.idle":"2024-03-28T04:13:46.334824Z","shell.execute_reply.started":"2024-03-28T04:13:46.154822Z","shell.execute_reply":"2024-03-28T04:13:46.334059Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#train AutoEncoder1\ndef train_e1d1():    \n    track_loss=0\n    \n    e1.train()\n    d1.train()\n    \n    for i,(x,_) in enumerate(train_dataloader):\n        x=torch.reshape(x,shape=(-1,784))\n        x=x.to(device)\n\n        latent=e1(x)\n        pred=d1(latent)\n            \n        loss=lossmse_fn(pred,x)\n        \n        track_loss+=loss.item()\n          \n        loss.backward()\n        \n        opte1.step()\n        optd1.step()\n        \n        opte1.zero_grad()\n        optd1.zero_grad()\n        \n        running_loss=track_loss/(i+(x.shape[0]/batch_size))\n            \n    return round(running_loss,4)\n\n#train AutoEncoder2\ndef train_e2d2():    \n    track_loss=0\n    \n    e2.train()\n    d2.train()\n    \n    for i,(x,_) in enumerate(train_dataloader):\n        x=torch.reshape(x,shape=(-1,784))\n        x=x.to(device)\n\n        latente1=e1(x)\n        latente2=e2(latente1.detach())\n        pred=d2(latente2)\n            \n        loss=lossmse_fn(pred,latente1.detach()) \n        \n        track_loss+=loss.item()\n        \n        loss.backward()\n        \n        opte2.step()\n        optd2.step()\n        opte2.zero_grad()\n        optd2.zero_grad()\n        \n        running_loss=track_loss/(i+(x.shape[0]/batch_size))\n    \n    return round(running_loss,4)\n\n#train Classifier\ndef train_clf():  \n    track_loss=0\n    num_correct=0\n    \n    clf.train()\n    \n    for i,(x,y) in enumerate(train_dataloader):\n        x=torch.reshape(x,shape=(-1,784))\n        x=x.to(device)\n        y=y.to(device)\n\n        latente1=e1(x)\n        latente2=e2(latente1)\n        pred=clf(latente2.detach())\n        \n        num_correct+=(torch.argmax(pred,dim=1)==y).type(torch.float).sum().item()\n            \n        loss=lossentropy_fn(pred,y)\n        \n        track_loss+=loss.item()\n        \n        loss.backward()\n        \n        optclf.step()\n        optclf.zero_grad()\n        \n        running_loss=track_loss/(i+(x.shape[0]/batch_size))\n        running_acc=(num_correct/((i*batch_size+x.shape[0])))*100\n    \n    return round(running_loss,4), round(running_acc,2)\n        \n#finetune whole \ndef train_whole():\n    track_loss=0\n    num_correct=0\n    \n    e1.train()\n    e2.train()\n    clf.train()\n    \n    for i,(x,y) in enumerate(train_dataloader):\n        x=torch.reshape(x,shape=(-1,784))\n        x=x.to(device)\n        y=y.to(device)\n\n        latente1=e1(x)\n        latente2=e2(latente1)\n        pred=clf(latente2)\n        \n        num_correct+=(torch.argmax(pred,dim=1)==y).type(torch.float).sum().item()\n            \n        loss=lossentropy_fn(pred,y)\n        \n        track_loss+=loss.item()\n        \n        loss.backward()\n        \n        opte1.step()\n        opte2.step()\n        optclf.step()\n        \n        opte1.zero_grad()\n        opte2.zero_grad()\n        optclf.zero_grad()\n        \n        running_loss=track_loss/(i+(x.shape[0]/batch_size))\n        running_acc=(num_correct/((i*batch_size+x.shape[0])))*100\n    \n    return round(running_loss,4),round(running_acc,2)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:13:46.336084Z","iopub.execute_input":"2024-03-28T04:13:46.336682Z","iopub.status.idle":"2024-03-28T04:13:46.357474Z","shell.execute_reply.started":"2024-03-28T04:13:46.336648Z","shell.execute_reply":"2024-03-28T04:13:46.356539Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Eval AutoEncoder1\ndef eval_e1d1():\n    track_loss=0\n    \n    e1.eval()\n    d1.eval()\n    \n    with torch.no_grad():\n        for i,(x,_) in enumerate(eval_dataloader):\n            x=torch.reshape(x,shape=(-1,784))\n            x=x.to(device)\n\n            latent=e1(x)\n            pred=d1(latent)\n\n            loss=lossmse_fn(pred,x)\n            track_loss+=loss.item()\n            \n            running_loss=track_loss/(i+(x.shape[0]/batch_size))\n    \n    return round(running_loss,4),x,pred\n\n#Eval AutoEncoder2\ndef eval_e2d2():\n    track_loss=0\n    \n    e2.eval()\n    d2.eval()\n    \n    with torch.no_grad():\n        for i,(x,_) in enumerate(eval_dataloader):\n            x=torch.reshape(x,shape=(-1,784))\n            x=x.to(device)\n\n            latente1=e1(x)\n            latente2=e2(latente1)\n            pred=d2(latente2)\n\n            loss=lossmse_fn(pred,latente1)\n            \n            track_loss+=loss.item()\n            \n            running_loss=track_loss/(i+(x.shape[0]/batch_size))\n    \n    return round(running_loss,4)\n            \n        \n#Eval Classifier\ndef eval_clf():\n    track_loss=0\n    num_correct=0\n    \n    clf.eval()\n    \n    with torch.no_grad():\n        for i,(x,y) in enumerate(eval_dataloader):\n            x=torch.reshape(x,shape=(-1,784))\n            x=x.to(device)\n            y=y.to(device)\n\n            latente1=e1(x)\n            latente2=e2(latente1)\n            pred=clf(latente2)\n            \n            num_correct+=(torch.argmax(pred,dim=1)==y).type(torch.float).sum().item()\n            running_acc=(num_correct/((i*batch_size+x.shape[0])))*100\n            \n            loss=lossentropy_fn(pred,y)\n            track_loss+=loss.item()\n            \n            running_loss=track_loss/(i+(x.shape[0]/batch_size))\n            \n    \n    return round(running_loss,4), round(running_acc,2)\n        \n#eval whole \ndef eval_whole():\n    track_loss=0\n    num_correct=0\n    \n    e1.eval()\n    e2.eval()\n    clf.eval()\n    with torch.no_grad():\n        for i,(x,y) in enumerate(eval_dataloader):\n            x=torch.reshape(x,shape=(-1,784))\n            x=x.to(device)\n            y=y.to(device)\n\n            latente1=e1(x)\n            latente2=e2(latente1)\n            pred=clf(latente2)\n            \n            num_correct+=(torch.argmax(pred,dim=1)==y).type(torch.float).sum().item()\n            running_acc=(num_correct/((i*batch_size+x.shape[0])))*100\n            \n            loss=lossentropy_fn(pred,y)\n            track_loss+=loss.item()\n            \n            running_loss=track_loss/(i+(x.shape[0]/batch_size))\n    \n    return round(running_loss,4), round(running_acc,2)","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:13:46.358546Z","iopub.execute_input":"2024-03-28T04:13:46.358825Z","iopub.status.idle":"2024-03-28T04:13:46.378385Z","shell.execute_reply.started":"2024-03-28T04:13:46.358801Z","shell.execute_reply":"2024-03-28T04:13:46.377481Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_epochs=20\n\nprint(\"-----------------------Auto Encoder 1------------------------\")\nfor i in range(n_epochs):\n    train_loss=train_e1d1()\n    eval_loss,x,pred=eval_e1d1()\n    print(\"Epoch=\", i+1,\", Train Loss=\",train_loss,\", Eval Loss=\",eval_loss,sep=\"\")\n    \nplt.figure(figsize=(3.2,2.4))\n\nr=torch.randint(low=0,high=pred.shape[0],size=(1,)).item()\n\nplt.subplot(1,2,1)\nplt.title(\"Orignal\")\nplt.imshow(torch.reshape(x[r],shape=(28,28)).cpu())\n\nplt.subplot(1,2,2)\nplt.title(\"Prediction\")\nplt.imshow(torch.reshape(pred[r],shape=(28,28)).cpu())\nplt.show()\n    \nprint(\"-----------------------Auto Encoder 2------------------------\")\nfor i in range(n_epochs):\n    print(\"Epoch=\", i+1,\", Train Loss=\",train_e2d2(),\", Eval Loss=\",eval_e2d2(),sep=\"\") \n\n\nprint(\"-----------------------Classifier Only------------------------\")\nfor i in range(n_epochs):\n    print(\"Epoch=\", i+1,\", Train Loss & Accuracy=\",train_clf(),\", Eval Loss & Accuracy=\",eval_clf(),sep=\"\") \n\n\nprint(\"--------------------Fine-Tuning Whole Metwork------------------\")\nfor i in range(n_epochs):\n    print(\"Epoch=\", i+1,\", Train Loss & Accuracy=\",train_whole(),\", Eval Loss & Accuracy=\",eval_whole(),sep=\"\") \n","metadata":{"execution":{"iopub.status.busy":"2024-03-28T04:13:46.379480Z","iopub.execute_input":"2024-03-28T04:13:46.379778Z","iopub.status.idle":"2024-03-28T04:26:15.348372Z","shell.execute_reply.started":"2024-03-28T04:13:46.379754Z","shell.execute_reply":"2024-03-28T04:26:15.347432Z"},"trusted":true},"outputs":[],"execution_count":null}]}