{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport random\ndevice = 'cuda' if torch.cuda.is_available else 'cpu'\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.metrics import accuracy_score\ntorch.manual_seed=42","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:30:53.722193Z","iopub.execute_input":"2024-06-19T02:30:53.722533Z","iopub.status.idle":"2024-06-19T02:30:58.893623Z","shell.execute_reply.started":"2024-06-19T02:30:53.722507Z","shell.execute_reply":"2024-06-19T02:30:58.892609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:30:58.895410Z","iopub.execute_input":"2024-06-19T02:30:58.895950Z","iopub.status.idle":"2024-06-19T02:31:02.522328Z","shell.execute_reply.started":"2024-06-19T02:30:58.895913Z","shell.execute_reply":"2024-06-19T02:31:02.521476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images.value_counts(subset=['label'])","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:02.523806Z","iopub.execute_input":"2024-06-19T02:31:02.524193Z","iopub.status.idle":"2024-06-19T02:31:02.548167Z","shell.execute_reply.started":"2024-06-19T02:31:02.524157Z","shell.execute_reply":"2024-06-19T02:31:02.547218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.concat([images[images['label']==1][:3795],\n           images[images['label']==2][:3795],\n           images[images['label']==3][:3795],\n           images[images['label']==4][:3795],\n           images[images['label']==5][:3795],\n           images[images['label']==6][:3795],\n           images[images['label']==7][:3795],\n           images[images['label']==8][:3795],\n           images[images['label']==9][:3795]\n          ])\ndf = df.sample(frac = 1)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:02.550311Z","iopub.execute_input":"2024-06-19T02:31:02.550619Z","iopub.status.idle":"2024-06-19T02:31:03.019902Z","shell.execute_reply.started":"2024-06-19T02:31:02.550593Z","shell.execute_reply":"2024-06-19T02:31:03.018840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(df)==3795*9","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:03.021331Z","iopub.execute_input":"2024-06-19T02:31:03.021720Z","iopub.status.idle":"2024-06-19T02:31:03.026567Z","shell.execute_reply.started":"2024-06-19T02:31:03.021684Z","shell.execute_reply":"2024-06-19T02:31:03.025636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:03.027917Z","iopub.execute_input":"2024-06-19T02:31:03.028255Z","iopub.status.idle":"2024-06-19T02:31:05.153677Z","shell.execute_reply.started":"2024-06-19T02:31:03.028224Z","shell.execute_reply":"2024-06-19T02:31:05.152811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self,df):\n        super().__init__()\n        self.data=df\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self,idx):\n        image = torch.FloatTensor(self.data[idx][1:])/255\n        label = self.data[idx][0]\n        return image.view(1,28,28),label","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:05.154880Z","iopub.execute_input":"2024-06-19T02:31:05.155664Z","iopub.status.idle":"2024-06-19T02:31:05.162144Z","shell.execute_reply.started":"2024-06-19T02:31:05.155627Z","shell.execute_reply":"2024-06-19T02:31:05.161173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_idx=int(len(df)*0.8)\nval_idx=int(len(df)*0.1)\n\ntrain_dataset=Dataset(df[:train_idx].values)\nval_dataset=Dataset(df[train_idx:train_idx+val_idx].values)\ntest_dataset=Dataset(df[train_idx+val_idx:].values)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:05.163643Z","iopub.execute_input":"2024-06-19T02:31:05.163994Z","iopub.status.idle":"2024-06-19T02:31:05.172305Z","shell.execute_reply.started":"2024-06-19T02:31:05.163961Z","shell.execute_reply":"2024-06-19T02:31:05.171462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.__getitem__(0)[0].shape","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:05.173701Z","iopub.execute_input":"2024-06-19T02:31:05.174090Z","iopub.status.idle":"2024-06-19T02:31:05.222026Z","shell.execute_reply.started":"2024-06-19T02:31:05.174058Z","shell.execute_reply":"2024-06-19T02:31:05.221183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader= torch.utils.data.DataLoader(train_dataset,batch_size=1000,shuffle=True)\nval_loader= torch.utils.data.DataLoader(val_dataset,batch_size=1000,shuffle=True)\ntest_loader= torch.utils.data.DataLoader(test_dataset,batch_size=1000,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:05.225101Z","iopub.execute_input":"2024-06-19T02:31:05.225397Z","iopub.status.idle":"2024-06-19T02:31:05.230500Z","shell.execute_reply.started":"2024-06-19T02:31:05.225372Z","shell.execute_reply":"2024-06-19T02:31:05.229631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classifier","metadata":{}},{"cell_type":"code","source":"class Classifier(nn.Module):\n    def __init__(self, output_dim):\n        super(Classifier, self).__init__()\n        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)\n        self.conv3 = nn.Conv2d(64, 128, kernel_size=3, padding=1)\n        self.conv4 = nn.Conv2d(128, 256, kernel_size=3, padding=1)\n        self.pool = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(256 * 1 * 1, 100)\n        self.fc2 = nn.Linear(100, output_dim)\n        self.actvfn = nn.ReLU()\n        self.flatten = nn.Flatten()\n\n    def forward(self, x):\n        x = self.pool(self.actvfn(self.conv1(x)))\n        x = self.pool(self.actvfn(self.conv2(x)))\n        x = self.pool(self.actvfn(self.conv3(x)))\n        x = self.pool(self.actvfn(self.conv4(x)))\n        x = self.flatten(x)\n        x = self.actvfn(self.fc1(x))\n        x = self.fc2(x)\n        return x\n\noutput_dim = 10\nclassifier_model=Classifier(output_dim)\nclassifier_model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:05.231609Z","iopub.execute_input":"2024-06-19T02:31:05.231901Z","iopub.status.idle":"2024-06-19T02:31:05.420801Z","shell.execute_reply.started":"2024-06-19T02:31:05.231878Z","shell.execute_reply":"2024-06-19T02:31:05.419905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_parameters = filter(lambda p: p.requires_grad, classifier_model.parameters())\nparams = sum([np.prod(p.size()) for p in model_parameters])\nparams","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:05.421897Z","iopub.execute_input":"2024-06-19T02:31:05.422177Z","iopub.status.idle":"2024-06-19T02:31:05.429457Z","shell.execute_reply.started":"2024-06-19T02:31:05.422153Z","shell.execute_reply":"2024-06-19T02:31:05.428470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:05.430839Z","iopub.execute_input":"2024-06-19T02:31:05.431191Z","iopub.status.idle":"2024-06-19T02:31:05.438662Z","shell.execute_reply.started":"2024-06-19T02:31:05.431159Z","shell.execute_reply":"2024-06-19T02:31:05.437803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.Adam(classifier_model.parameters(), lr=0.001)\n\nnum_epochs = 20\n\nclassifier_model.train()\n\nfor epoch in range(num_epochs):\n    total_loss=0\n    total_val_loss=0\n    train_preds = []\n    train_labels = []\n    best_val_loss = float('inf')\n    best_train_loss = float('inf')\n    train_losses, val_losses=[],[]\n    \n    print('Training...')\n    for batch in tqdm(train_loader):\n        inputs,labels = batch\n        inputs= inputs.to(device)\n        labels = torch.LongTensor(labels).to(device)\n        outputs = classifier_model(inputs)\n\n        loss = nn.functional.cross_entropy(outputs, labels)\n        total_loss += loss.item()\n        train_preds.extend(torch.argmax(outputs, dim=1).cpu().numpy())\n        train_labels.extend(labels.cpu().numpy())\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    train_accuracy = accuracy_score(train_labels, train_preds)\n    total_loss=total_loss/len(train_loader)\n    train_losses.append(total_loss)\n    \n    classifier_model.eval()\n    all_preds = []\n    all_labels = []\n    \n    print('Validating...')\n    with torch.no_grad():\n        for val_batch in val_loader:\n            val_inputs, val_labels = val_batch\n            val_inputs = val_inputs.to(device)\n            val_labels = torch.LongTensor(val_labels).to(device)\n\n            val_outputs = classifier_model(val_inputs)\n            val_loss = nn.functional.cross_entropy(val_outputs, val_labels)\n            total_val_loss+=val_loss\n            all_preds.extend(torch.argmax(val_outputs, dim=1).cpu().numpy())\n            all_labels.extend(val_labels.cpu().numpy())\n\n    val_accuracy = accuracy_score(all_labels, all_preds)\n    total_val_loss=total_val_loss/(len(val_loader))\n    val_losses.append(total_val_loss)\n    if total_val_loss < best_val_loss:\n            best_val_loss = total_val_loss\n            best_train_loss = total_loss\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': classifier_model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'loss': loss\n            }, './CNNClassifier.pt')\n    if (epoch + 1) % 10 == 0:\n        print(f'Epoch [{epoch+1}/{num_epochs}] | Training Loss: {total_loss:.4f} | Validation Loss:{total_val_loss} | Training Accuracy: {train_accuracy:.4f} | Validation Accuracy:{val_accuracy}')","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:05.439947Z","iopub.execute_input":"2024-06-19T02:31:05.440207Z","iopub.status.idle":"2024-06-19T02:31:38.128542Z","shell.execute_reply.started":"2024-06-19T02:31:05.440185Z","shell.execute_reply":"2024-06-19T02:31:38.127591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load(\"./CNNClassifier.pt\")\nclassifier_model.load_state_dict(checkpoint['model_state_dict'])\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])\nepoch = checkpoint['epoch']\nloss = checkpoint['loss']\n\nclassifier_model.eval()\nclassifier_model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:38.129929Z","iopub.execute_input":"2024-06-19T02:31:38.130568Z","iopub.status.idle":"2024-06-19T02:31:38.151658Z","shell.execute_reply.started":"2024-06-19T02:31:38.130532Z","shell.execute_reply":"2024-06-19T02:31:38.150826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_model.eval()\nall_preds = []\ntotal_test_loss=0\nall_labels = []\nwith torch.no_grad():\n    for test_batch in test_loader:\n        test_inputs, test_labels = test_batch\n        test_inputs = test_inputs.to(device)\n        test_labels = torch.LongTensor(test_labels).to(device)\n\n        test_outputs = classifier_model(test_inputs)\n        test_loss = nn.functional.cross_entropy(test_outputs, test_labels)\n        total_test_loss+=test_loss\n        all_preds.extend(torch.argmax(test_outputs, dim=1).cpu().numpy())\n        all_labels.extend(test_labels.cpu().numpy())\n\ntest_accuracy = accuracy_score(all_labels, all_preds)\ntotal_test_loss=total_test_loss/(len(test_loader))\nprint(f\"Test Accuracy:{test_accuracy}| Test Loss:{total_test_loss}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-19T02:31:38.152712Z","iopub.execute_input":"2024-06-19T02:31:38.152981Z","iopub.status.idle":"2024-06-19T02:31:38.322719Z","shell.execute_reply.started":"2024-06-19T02:31:38.152958Z","shell.execute_reply":"2024-06-19T02:31:38.321769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Adversarial Model","metadata":{}},{"cell_type":"code","source":"idx=0\nAdversarialLabel = None\n\nwhile AdversarialLabel !=1:\n    AdversarialLabel = test_dataset.__getitem__(idx)[1]\n    idx+=1\nassert AdversarialLabel==1\n\nAdversarialImage = test_dataset.__getitem__(idx-1)[0].unsqueeze(0).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T04:17:02.602853Z","iopub.execute_input":"2024-06-19T04:17:02.603317Z","iopub.status.idle":"2024-06-19T04:17:02.611158Z","shell.execute_reply.started":"2024-06-19T04:17:02.603253Z","shell.execute_reply":"2024-06-19T04:17:02.610124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\ndef plot_mnist_image(image):\n    plt.imshow(image, cmap='gray')\n    plt.axis('off')\n    plt.show()\n\nplot_mnist_image(AdversarialImage.detach().cpu().numpy().reshape(28,28))","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:27:45.155874Z","iopub.execute_input":"2024-06-19T03:27:45.156241Z","iopub.status.idle":"2024-06-19T03:27:45.241020Z","shell.execute_reply.started":"2024-06-19T03:27:45.156209Z","shell.execute_reply":"2024-06-19T03:27:45.239569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AdversarialImage.requires_grad = True\ntarget=torch.LongTensor([4]).to(device) if AdversarialLabel!=8 else torch.LongTensor([4]).to(device)\n\nclassifier_model.to(device)\nclassifier_model.eval()\n\nadv_optimizer = optim.Adam([AdversarialImage], lr=0.001)\nadv_criterion = nn.MSELoss()\n\n# Adversarial training loop\nnum_epochs = 1000\nadversarial_training_losses=[]\nfor epoch in range(num_epochs):\n    output = classifier_model(AdversarialImage)\n    \n    if torch.argmax(output).item()==target:\n        print(f'-----\\nEpoch [{epoch+1}/{num_epochs}] | Training Loss: {adv_loss:.4f} | output class: {torch.argmax(output).item()}')\n        break\n    adv_loss = nn.functional.cross_entropy(output,target) \n    adversarial_training_losses.append(adv_loss.item())\n    adv_optimizer.zero_grad()\n    adv_loss.backward()\n    adv_optimizer.step()\n\n    print(f'Epoch [{epoch+1}/{num_epochs}] | Training Loss: {adv_loss:.4f} | output class: {torch.argmax(output).item()}')","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:28:09.209011Z","iopub.execute_input":"2024-06-19T03:28:09.209499Z","iopub.status.idle":"2024-06-19T03:28:09.561867Z","shell.execute_reply.started":"2024-06-19T03:28:09.209441Z","shell.execute_reply":"2024-06-19T03:28:09.560818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Epochs = 85\n\nLoss-Avg = 0.70","metadata":{}},{"cell_type":"code","source":"plt.plot(adversarial_training_losses)\nplt.title('Adversarial Training Loss Curve')\nplt.ylabel(\"Loss\")\nplt.xlabel(\"Epochs\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:28:15.903296Z","iopub.execute_input":"2024-06-19T03:28:15.903693Z","iopub.status.idle":"2024-06-19T03:28:16.180169Z","shell.execute_reply.started":"2024-06-19T03:28:15.903662Z","shell.execute_reply":"2024-06-19T03:28:16.179328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(torch.argmax(classifier_model(test_dataset.__getitem__(idx-1)[0].unsqueeze(0).to(device))).item())\nprint(torch.argmax(classifier_model(AdversarialImage)).item())","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:28:19.960016Z","iopub.execute_input":"2024-06-19T03:28:19.960365Z","iopub.status.idle":"2024-06-19T03:28:19.969667Z","shell.execute_reply.started":"2024-06-19T03:28:19.960339Z","shell.execute_reply":"2024-06-19T03:28:19.968492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\ndef plot_mnist_image(image):\n    plt.imshow(image, cmap='gray')\n    plt.axis('off')\n    plt.show()\n\nplot_mnist_image(AdversarialImage.detach().cpu().numpy().reshape(28,28))","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:28:22.807356Z","iopub.execute_input":"2024-06-19T03:28:22.808220Z","iopub.status.idle":"2024-06-19T03:28:22.879748Z","shell.execute_reply.started":"2024-06-19T03:28:22.808189Z","shell.execute_reply":"2024-06-19T03:28:22.878816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Encoder","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nclass Encoder(nn.Module):\n    def __init__(self):\n        super(Encoder, self).__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1) # Output: 16 x 14 x 14\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, stride=2, padding=1) # Output: 32 x 7 x 7\n        self.conv3 = nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1) # Output: 64 x 4 x 4\n        self.fc = nn.Linear(64 * 4 * 4, 100)\n        self.actvfn = nn.ReLU()\n\n    def forward(self, x):\n        x = self.actvfn(self.conv1(x))\n        x = self.actvfn(self.conv2(x))\n        x = self.actvfn(self.conv3(x))\n        x = x.view(x.size(0), -1) # Flatten\n        x = self.fc(x)\n        return x\n\nclass Decoder(nn.Module):\n    def __init__(self):\n        super(Decoder, self).__init__()\n        self.fc1 = nn.Linear(100, 64 * 4 * 4)\n        self.deconv1 = nn.ConvTranspose2d(64, 32, kernel_size=3, stride=2, padding=1, output_padding=0) # Output: 32 x 7 x 7\n        self.deconv2 = nn.ConvTranspose2d(32, 16, kernel_size=3, stride=2, padding=1, output_padding=1) # Output: 16 x 14 x 14\n        self.deconv3 = nn.ConvTranspose2d(16, 1, kernel_size=3, stride=2, padding=1, output_padding=1)  # Output: 1 x 28 x 28\n        self.actvfn = nn.ReLU()\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        x = self.actvfn(self.fc1(x))\n        x = x.view(x.size(0), 64, 4, 4) # Reshape for deconvolution\n        x = self.actvfn(self.deconv1(x))\n        x = self.actvfn(self.deconv2(x))\n        x = self.sigmoid(self.deconv3(x)) # Use sigmoid to ensure output is between 0 and 1\n        return x\n\nclass Autoencoder(nn.Module):\n    def __init__(self):\n        super(Autoencoder, self).__init__()\n        self.encoder = Encoder()\n        self.decoder = Decoder()\n\n    def forward(self, x):\n        latent = self.encoder(x)\n        reconstructed = self.decoder(latent)\n        return latent, reconstructed\n\nautoencoder = Autoencoder().to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.Adam(autoencoder.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:29:24.675137Z","iopub.execute_input":"2024-06-19T03:29:24.675537Z","iopub.status.idle":"2024-06-19T03:29:24.697880Z","shell.execute_reply.started":"2024-06-19T03:29:24.675501Z","shell.execute_reply":"2024-06-19T03:29:24.696924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 200\nepochs,train_losses, val_losses=[],[],[]\n\nfor epoch in range(num_epochs):\n    epochs.append(epoch+200+1)\n    autoencoder.train()\n    total_loss = 0\n    best_val_loss = float('inf')\n    best_train_loss = float('inf')\n    for batch in train_loader:\n        inputs, _ = batch\n        inputs = inputs.to(device)\n        latent, outputs = autoencoder(inputs)\n        loss = criterion(outputs, inputs) # Compare the outputs directly to inputs\n        total_loss += loss.item()\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    total_loss /= len(train_loader)\n    train_losses.append(total_loss)\n    autoencoder.eval()\n    val_total_loss = 0\n    with torch.no_grad():\n        for val_batch in val_loader:\n            val_inputs, _ = val_batch\n            val_inputs = val_inputs.to(device)\n            _, val_outputs = autoencoder(val_inputs)\n\n            val_loss = criterion(val_outputs, val_inputs)\n            val_total_loss += val_loss.item()\n        val_total_loss /= len(val_loader)\n        val_losses.append(val_total_loss)\n    if val_total_loss < best_val_loss:\n        best_val_loss = val_total_loss\n        best_train_loss = total_loss\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': autoencoder.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'loss': loss\n        }, './Autoencoder.pt')\n    if (epoch + 1) % 10 == 0:\n        print(f'Epoch [{epoch+1}/{num_epochs}], Train Loss: {total_loss:.4f} | Validation Loss: {val_total_loss:.4f}')","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:29:28.894698Z","iopub.execute_input":"2024-06-19T03:29:28.895320Z","iopub.status.idle":"2024-06-19T03:34:05.811377Z","shell.execute_reply.started":"2024-06-19T03:29:28.895289Z","shell.execute_reply":"2024-06-19T03:34:05.810329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nplt.plot(range(1, num_epochs + 1), train_losses, label='Training Loss')\nplt.plot(range(1, num_epochs + 1), val_losses, label='Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Training and Validation Losses')\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:34:05.813421Z","iopub.execute_input":"2024-06-19T03:34:05.814094Z","iopub.status.idle":"2024-06-19T03:34:06.165710Z","shell.execute_reply.started":"2024-06-19T03:34:05.814059Z","shell.execute_reply":"2024-06-19T03:34:06.164784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load(\"./Autoencoder.pt\")\nautoencoder.load_state_dict(checkpoint['model_state_dict'])\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])\nepoch = checkpoint['epoch']\nloss = checkpoint['loss']\n\nautoencoder.eval()\nautoencoder.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:34:22.811213Z","iopub.execute_input":"2024-06-19T03:34:22.811588Z","iopub.status.idle":"2024-06-19T03:34:22.837410Z","shell.execute_reply.started":"2024-06-19T03:34:22.811561Z","shell.execute_reply":"2024-06-19T03:34:22.836441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder.eval()\nwith torch.no_grad():\n    test_total_loss=0\n    for test_batch in test_loader:\n        test_inputs,test_labels = test_batch\n        test_inputs= test_inputs.to(device)\n        test_outputs = autoencoder(test_inputs)\n\n        test_loss = criterion(test_outputs[1], test_inputs)\n        test_total_loss += test_loss.item()\n        test_total_loss=test_total_loss/len(test_loader)\n    print(test_total_loss)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:34:27.290401Z","iopub.execute_input":"2024-06-19T03:34:27.290779Z","iopub.status.idle":"2024-06-19T03:34:27.435432Z","shell.execute_reply.started":"2024-06-19T03:34:27.290753Z","shell.execute_reply":"2024-06-19T03:34:27.434463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nautoencoder.eval()\nj=100\nwith torch.no_grad():\n    inputs,labels = test_dataset.__getitem__(j)\n    inputs= inputs.unsqueeze(0).to(device)\n    print(inputs.shape)\n    outputs = autoencoder(inputs)\n\nn = 1  \nplt.figure(figsize=(20, 4))\nfor i in range(n):\n    # Display original\n    ax = plt.subplot(2, n, i + 1)\n    plt.imshow(test_dataset.__getitem__(j)[0].detach().numpy().reshape(28, 28))\n    plt.gray()\n    print(\"Original Item\")\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\n    plt.show()\n\n    # Display reconstruction\n    print(\"Reconstructed Item\")\n    ax = plt.subplot(2, n, i + 1 + n)\n    plt.imshow(outputs[1].detach().cpu().reshape(28, 28))\n    plt.gray()\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-19T03:34:32.525195Z","iopub.execute_input":"2024-06-19T03:34:32.526280Z","iopub.status.idle":"2024-06-19T03:34:32.628670Z","shell.execute_reply.started":"2024-06-19T03:34:32.526226Z","shell.execute_reply":"2024-06-19T03:34:32.627588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using Latent Features as Inputs","metadata":{}},{"cell_type":"code","source":"class Latent_Classifier(nn.Module):\n    def __init__(self, dim,output_dim):\n        super(Latent_Classifier, self).__init__()\n        self.classifier_layer=nn.Linear(dim, 200)\n        self.classifier_layer_2=nn.Linear(200, output_dim)\n        self.actvfn=nn.ReLU()\n        \n    def forward(self,x):\n        out = self.actvfn(self.classifier_layer(x))\n        out = self.actvfn(self.classifier_layer_2(out))\n        return out\n    \nlatent_model=Latent_Classifier(100,10)\nlatent_model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T04:59:33.347799Z","iopub.execute_input":"2024-06-19T04:59:33.348177Z","iopub.status.idle":"2024-06-19T04:59:33.361397Z","shell.execute_reply.started":"2024-06-19T04:59:33.348147Z","shell.execute_reply":"2024-06-19T04:59:33.360315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_parameters = filter(lambda p: p.requires_grad, latent_model.parameters())\nparams = sum([np.prod(p.size()) for p in model_parameters])\nparams","metadata":{"execution":{"iopub.status.busy":"2024-06-19T04:51:32.058155Z","iopub.execute_input":"2024-06-19T04:51:32.058571Z","iopub.status.idle":"2024-06-19T04:51:32.066896Z","shell.execute_reply.started":"2024-06-19T04:51:32.058539Z","shell.execute_reply":"2024-06-19T04:51:32.065860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder.eval()\nlatent_optimizer = optim.Adam(latent_model.parameters(), lr=0.001)\nepochs,train_losses, val_losses=[],[],[]\nnum_epochs = 200\n\nfor epoch in range(num_epochs):\n    latent_model.train()\n    total_loss=0\n    total_val_loss=0\n    best_val_loss = float('inf')\n    best_train_loss = float('inf')\n    train_preds = []\n    train_labels = []\n    for batch in train_loader:\n        inputs,labels = batch\n        inputs= inputs.to(device)\n        latent_inputs=autoencoder(inputs)\n        labels = torch.LongTensor(labels).to(device)\n        outputs = latent_model(latent_inputs[0])\n\n        loss = nn.functional.cross_entropy(outputs, labels)\n        total_loss += loss.item()\n        train_preds.extend(torch.argmax(outputs, dim=1).cpu().numpy())\n        train_labels.extend(labels.cpu().numpy())\n        latent_optimizer.zero_grad()\n        loss.backward()\n        latent_optimizer.step()\n    train_accuracy = accuracy_score(train_labels, train_preds)\n    total_loss=total_loss/len(train_loader)\n    train_losses.append(total_loss)\n    \n    latent_model.eval()\n    all_preds = []\n    all_labels = []\n    with torch.no_grad():\n        for val_batch in val_loader:\n            val_inputs, val_labels = val_batch\n            val_inputs = val_inputs.to(device)\n            latent_val_inputs = autoencoder(val_inputs)\n            val_labels = torch.LongTensor(val_labels).to(device)\n            val_outputs = latent_model(latent_val_inputs[0])\n            val_loss = nn.functional.cross_entropy(val_outputs, val_labels)\n            total_val_loss+=val_loss.item()\n            all_preds.extend(torch.argmax(val_outputs, dim=1).cpu().numpy())\n            all_labels.extend(val_labels.cpu().numpy())\n\n    val_accuracy = accuracy_score(all_labels, all_preds)\n    total_val_loss=total_val_loss/(len(val_loader))\n    val_losses.append(total_val_loss)\n    \n    if val_total_loss < best_val_loss:\n        best_val_loss = val_total_loss\n        best_train_loss = total_loss\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': latent_model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'loss': loss\n        }, './Latent_model.pt')\n    if (epoch + 1) % 10 == 0:\n        print(f'Epoch [{epoch+1}/{num_epochs}] | Training Loss: {total_loss:.4f} | Validation Loss:{total_val_loss} | Training Accuracy: {train_accuracy:.4f} | Validation Accuracy:{val_accuracy}')","metadata":{"execution":{"iopub.status.busy":"2024-06-19T04:59:47.496303Z","iopub.execute_input":"2024-06-19T04:59:47.497242Z","iopub.status.idle":"2024-06-19T05:00:28.610953Z","shell.execute_reply.started":"2024-06-19T04:59:47.497207Z","shell.execute_reply":"2024-06-19T05:00:28.609468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nplt.plot(train_losses, label='Training Loss')\nplt.plot(val_losses, label='Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Training and Validation Losses For 1 hidden Layer')\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-19T04:57:27.108863Z","iopub.execute_input":"2024-06-19T04:57:27.109273Z","iopub.status.idle":"2024-06-19T04:57:27.438465Z","shell.execute_reply.started":"2024-06-19T04:57:27.109228Z","shell.execute_reply":"2024-06-19T04:57:27.437465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load(\"./Latent_model.pt\")\nlatent_model.load_state_dict(checkpoint['model_state_dict'])\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])\nepoch = checkpoint['epoch']\nloss = checkpoint['loss']\n\nlatent_model.eval()\nlatent_model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T05:00:34.355184Z","iopub.execute_input":"2024-06-19T05:00:34.355605Z","iopub.status.idle":"2024-06-19T05:00:34.378650Z","shell.execute_reply.started":"2024-06-19T05:00:34.355572Z","shell.execute_reply":"2024-06-19T05:00:34.377590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch","metadata":{"execution":{"iopub.status.busy":"2024-06-19T04:57:36.062118Z","iopub.execute_input":"2024-06-19T04:57:36.062536Z","iopub.status.idle":"2024-06-19T04:57:36.069402Z","shell.execute_reply.started":"2024-06-19T04:57:36.062507Z","shell.execute_reply":"2024-06-19T04:57:36.068323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"latent_model.eval()\nall_preds = []\ntotal_test_loss=0\nall_labels = []\nwith torch.no_grad():\n    for test_batch in test_loader:\n        test_inputs, test_labels = test_batch\n        test_inputs = test_inputs.to(device)\n        test_labels = torch.LongTensor(test_labels).to(device)\n        latent_test_inputs = autoencoder(test_inputs)\n        test_outputs = latent_model(latent_test_inputs[0])\n        test_loss = nn.functional.cross_entropy(test_outputs, test_labels)\n        total_test_loss+=test_loss\n        all_preds.extend(torch.argmax(test_outputs, dim=1).cpu().numpy())\n        all_labels.extend(test_labels.cpu().numpy())\n\ntest_accuracy = accuracy_score(all_labels, all_preds)\ntotal_test_loss=total_test_loss/(len(test_loader))\nprint(f\"Test Accuracy:{test_accuracy}| Test Loss:{total_test_loss}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-19T05:00:38.849056Z","iopub.execute_input":"2024-06-19T05:00:38.850044Z","iopub.status.idle":"2024-06-19T05:00:39.025110Z","shell.execute_reply.started":"2024-06-19T05:00:38.850007Z","shell.execute_reply":"2024-06-19T05:00:39.023915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder.eval()\nAdversarialILatent=torch.FloatTensor(autoencoder(AdversarialImage)[0].detach().cpu().numpy()).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-19T05:00:43.549755Z","iopub.execute_input":"2024-06-19T05:00:43.550188Z","iopub.status.idle":"2024-06-19T05:00:43.558251Z","shell.execute_reply.started":"2024-06-19T05:00:43.550157Z","shell.execute_reply":"2024-06-19T05:00:43.556900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AdversarialILatent","metadata":{"execution":{"iopub.status.busy":"2024-06-19T05:00:46.245372Z","iopub.execute_input":"2024-06-19T05:00:46.245786Z","iopub.status.idle":"2024-06-19T05:00:46.257445Z","shell.execute_reply.started":"2024-06-19T05:00:46.245751Z","shell.execute_reply":"2024-06-19T05:00:46.256284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AdversarialILatent.requires_grad = True\ntarget=torch.LongTensor([8]).to(device)\n\nlatent_model.to(device)\nlatent_model.eval()\n\nadv_optimizer = optim.Adam([AdversarialILatent], lr=0.001)\nadv_criterion = nn.MSELoss()\n\n# Adversarial training loop\nnum_epochs = 1000\nadversarial_training_losses=[]\nfor epoch in range(num_epochs):\n    output = latent_model(AdversarialILatent)\n    \n    if torch.argmax(output).item()==target:\n        print(f'-----\\nEpoch [{epoch+1}/{num_epochs}] | Training Loss: {adv_loss:.4f} | output class: {torch.argmax(output).item()}')\n        break\n    adv_loss = nn.functional.cross_entropy(output,target) \n    adversarial_training_losses.append(adv_loss.item())\n    adv_optimizer.zero_grad()\n    adv_loss.backward()\n    adv_optimizer.step()\n\n    if (epoch+1)%10==0:\n        print(f'Epoch [{epoch+1}/{num_epochs}] | Training Loss: {adv_loss:.4f} | output class: {torch.argmax(output).item()}')","metadata":{"execution":{"iopub.status.busy":"2024-06-19T05:01:20.991726Z","iopub.execute_input":"2024-06-19T05:01:20.992731Z","iopub.status.idle":"2024-06-19T05:01:21.371104Z","shell.execute_reply.started":"2024-06-19T05:01:20.992694Z","shell.execute_reply":"2024-06-19T05:01:21.369907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(adversarial_training_losses)\nplt.title('Adversarial Training Loss Curve')\nplt.ylabel(\"Loss\")\nplt.xlabel(\"Epochs\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-19T05:01:26.848920Z","iopub.execute_input":"2024-06-19T05:01:26.849308Z","iopub.status.idle":"2024-06-19T05:01:27.119794Z","shell.execute_reply.started":"2024-06-19T05:01:26.849272Z","shell.execute_reply":"2024-06-19T05:01:27.118768Z"},"trusted":true},"execution_count":null,"outputs":[]}]}