{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nfrom torchvision import datasets, utils, transforms\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch import nn\nfrom pytorch_lightning import LightningModule, Trainer\nfrom pytorch_lightning.utilities.model_summary import ModelSummary\nfrom sklearn.model_selection import train_test_split\nimport torchvision.transforms\nfrom torchmetrics import Accuracy\n!pip install torchsummary\nfrom torchsummary import summary\nimport matplotlib.pyplot as plt\nimport numpy\nfrom sklearn.manifold import TSNE\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import plot_confusion_matrix\nfrom sklearn.metrics import classification_report\nimport gc","metadata":{"id":"6kFqQrXuTN9O","execution":{"iopub.status.busy":"2022-09-21T18:40:03.852907Z","iopub.execute_input":"2022-09-21T18:40:03.853671Z","iopub.status.idle":"2022-09-21T18:40:13.298761Z","shell.execute_reply.started":"2022-09-21T18:40:03.853633Z","shell.execute_reply":"2022-09-21T18:40:13.297588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Model, Train, Test, Predict, Dataloaders etc., with Lightning Module","metadata":{}},{"cell_type":"code","source":"import torchvision.models as models\n\nclass VGGNET(LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.data_dir = \"/kaggle/temp\"\n        self.batch_size = 512 if torch.cuda.is_available() else 128\n        self.in_features = 784\n        self.lr = 1e-2\n        \n        self.train_accuracy = Accuracy()\n        self.val_accuracy = Accuracy()\n        self.test_accuracy = Accuracy()\n        self.criterion = nn.CrossEntropyLoss()\n\n        self.model = models.vgg16_bn()\n        self.model.features[0] = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n        self.model.classifier[6] = nn.Linear(in_features=4096, out_features=10, bias=True)\n\n        self.tranforms = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize((0.5,), (0.5,)),\n        ])\n\n    def summary(self, input):\n        return summary(self.cuda(), input) if torch.cuda.is_available() else summary(self, input)\n        \n    def prepare_data(self):\n        datasets.CIFAR10(root=self.data_dir, train=True, download=True)\n        datasets.CIFAR10(root=self.data_dir, train=False, download=True)\n        \n    def setup(self, stage=None):\n        if stage == \"fit\" or stage is None:\n            mnist_train = datasets.CIFAR10(root=self.data_dir, train=True, transform=self.tranforms)\n            mnist_train = DataLoader(mnist_train)\n            train_dataset, val_dataset = train_test_split(mnist_train.dataset, stratify=mnist_train.dataset.targets,test_size=0.20)\n\n            self.mnist_train = train_dataset\n            self.mnist_val = val_dataset\n\n        if stage == \"test\" or stage is None:\n            self.mnist_test = datasets.CIFAR10(root=self.data_dir, train=False, transform=self.tranforms)\n            \n    def train_dataloader(self):\n        return DataLoader(self.mnist_train, batch_size=self.batch_size, shuffle=True)\n\n    def val_dataloader(self):\n        return DataLoader(self.mnist_val, batch_size=self.batch_size)\n\n    def test_dataloader(self):\n        return DataLoader(self.mnist_test, batch_size=self.batch_size)\n\n    def predict_dataloader(self):\n        return DataLoader(self.mnist_test, batch_size=self.batch_size)\n    \n    def configure_optimizers(self):\n        optimizer = torch.optim.SGD(\n            self.model.parameters(),\n            lr=self.lr,\n            momentum=0.9\n        )\n        lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer,\n            factor=.1,\n            patience=5,\n            verbose=True,\n        )\n        return {\n            \"optimizer\": optimizer,\n            \"lr_scheduler\": lr_scheduler,\n            \"monitor\": \"train_loss\",\n        }\n    \n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch, batch_idx):\n        image, label = batch\n        pred = self(image)\n        loss = self.criterion(pred, label)\n        self.train_accuracy(pred, label)\n\n        self.log(\"train_loss\", loss)\n        self.log(\"train_acc\", self.train_accuracy)\n\n        return {\"loss\": loss}\n\n    def validation_step(self, batch, batch_idx):\n        image, label = batch\n        pred = self(image)\n        loss = self.criterion(pred, label)\n        self.val_accuracy(pred, label)\n\n        self.log(\"val_loss\", loss, prog_bar=True)\n        self.log(\"val_acc\", self.val_accuracy, prog_bar=True)\n\n    def test_step(self, batch, batch_idx):\n        image, label = batch\n        pred = self(image)\n        loss = self.criterion(pred, label)\n        self.test_accuracy(pred, label)\n\n        self.log(\"test_loss\", loss)\n        self.log(\"test_acc\", self.test_accuracy)\n\n    def predict_step(self, batch, batch_idx):\n        image, label = batch\n        features = self.model.classifier[:-3](torch.flatten(self.model.avgpool(self.model.features(image)), start_dim=1))\n\n        pred = self(image)\n\n        pred = torch.argmax(pred, 1)\n\n        return label, features, pred","metadata":{"id":"am3gAbS7TN9Q","execution":{"iopub.status.busy":"2022-09-21T18:40:13.301551Z","iopub.execute_input":"2022-09-21T18:40:13.301956Z","iopub.status.idle":"2022-09-21T18:40:13.324930Z","shell.execute_reply.started":"2022-09-21T18:40:13.301914Z","shell.execute_reply":"2022-09-21T18:40:13.323929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setup Lightning Trainer","metadata":{}},{"cell_type":"code","source":"from pytorch_lightning.loggers import CSVLogger\nfrom pytorch_lightning.callbacks.progress import TQDMProgressBar\n\nexperiment = VGGNET()\n\ntrainer = Trainer(\n    default_root_dir=\"logs/\",\n    accelerator=\"auto\",\n    devices=1 if torch.cuda.is_available() else None,\n    max_epochs=100,\n    logger=CSVLogger(save_dir=\"logs/\"),\n    callbacks=[TQDMProgressBar(refresh_rate=20)],\n)","metadata":{"id":"-l9c80T8TN9R","outputId":"0e853e2b-15b9-4d84-c8a2-4bc1b17391d7","execution":{"iopub.status.busy":"2022-09-21T18:40:13.326836Z","iopub.execute_input":"2022-09-21T18:40:13.327231Z","iopub.status.idle":"2022-09-21T18:40:16.554758Z","shell.execute_reply.started":"2022-09-21T18:40:13.327195Z","shell.execute_reply":"2022-09-21T18:40:16.553446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment.prepare_data()\nexperiment.setup()\noriginal_batch_size = experiment.batch_size\nexperiment.batch_size = 64\ntrain_loader = experiment.train_dataloader()\nexperiment.batch_size = original_batch_size","metadata":{"id":"Qa4rmqVEDYRm","outputId":"419bde2a-73da-4542-b755-0b4abb641523","execution":{"iopub.status.busy":"2022-09-21T18:40:16.558378Z","iopub.execute_input":"2022-09-21T18:40:16.558990Z","iopub.status.idle":"2022-09-21T18:40:33.003232Z","shell.execute_reply.started":"2022-09-21T18:40:16.558947Z","shell.execute_reply":"2022-09-21T18:40:33.002235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot a image batch","metadata":{}},{"cell_type":"code","source":"image, label = next(iter(train_loader))\n\nprint(image.shape)\ngrid = utils.make_grid(image, padding=0)\nfig, ax = plt.subplots(1,1, dpi=200)\nax.imshow(grid.permute(1,2,0))\nax.axis(\"off\")\n\ndel train_loader\ndel image\ndel label\ndel grid","metadata":{"id":"pyqrpbdsEEAl","outputId":"c93ca27d-42bb-4ee9-ea2f-97d0c917ffa2","execution":{"iopub.status.busy":"2022-09-21T18:40:33.004566Z","iopub.execute_input":"2022-09-21T18:40:33.004972Z","iopub.status.idle":"2022-09-21T18:40:33.391410Z","shell.execute_reply.started":"2022-09-21T18:40:33.004934Z","shell.execute_reply":"2022-09-21T18:40:33.390575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summarize Model","metadata":{}},{"cell_type":"code","source":"experiment.summary((3, 32, 32))","metadata":{"id":"4mrLLPpP0V4W","outputId":"9244262d-c59d-4673-e772-c70e8abbd881","execution":{"iopub.status.busy":"2022-09-21T18:40:33.392663Z","iopub.execute_input":"2022-09-21T18:40:33.394218Z","iopub.status.idle":"2022-09-21T18:40:34.600089Z","shell.execute_reply.started":"2022-09-21T18:40:33.394181Z","shell.execute_reply":"2022-09-21T18:40:34.598749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fit model","metadata":{}},{"cell_type":"code","source":"trainer.fit(experiment)","metadata":{"id":"-P-78H8YTN9S","outputId":"058f28fa-890e-4f95-9ab9-d89bd333dab4","execution":{"iopub.status.busy":"2022-09-21T18:40:34.601581Z","iopub.execute_input":"2022-09-21T18:40:34.603166Z","iopub.status.idle":"2022-09-21T19:00:04.439541Z","shell.execute_reply.started":"2022-09-21T18:40:34.603122Z","shell.execute_reply":"2022-09-21T19:00:04.438446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test report","metadata":{}},{"cell_type":"code","source":"trainer.test(experiment)","metadata":{"id":"ubkf1K8egnzl","outputId":"6c77671c-3262-438f-9cc4-7a9563b5b10a","execution":{"iopub.status.busy":"2022-09-21T19:00:04.441686Z","iopub.execute_input":"2022-09-21T19:00:04.442110Z","iopub.status.idle":"2022-09-21T19:00:12.466790Z","shell.execute_reply.started":"2022-09-21T19:00:04.442070Z","shell.execute_reply":"2022-09-21T19:00:12.464749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classification Report","metadata":{}},{"cell_type":"code","source":"predicts = trainer.predict(experiment)","metadata":{"id":"DBydv5XdZjUu","outputId":"00362678-f4cd-4d9f-fa4a-4003485a269e","execution":{"iopub.status.busy":"2022-09-21T19:00:12.468939Z","iopub.execute_input":"2022-09-21T19:00:12.469410Z","iopub.status.idle":"2022-09-21T19:00:21.448974Z","shell.execute_reply.started":"2022-09-21T19:00:12.469364Z","shell.execute_reply":"2022-09-21T19:00:21.447920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"truth_labels = []\n_ = [truth_labels.extend(value[0]) for value in predicts]\n\nfeatures = []\n_ = [features.extend(value[1].tolist()) for value in predicts]\nfeatures = numpy.array(features)\n\npred_labels = []\n_ = [pred_labels.extend(value[2]) for value in predicts]\n\ndel predicts","metadata":{"id":"fWTjMDp8anDB","execution":{"iopub.status.busy":"2022-09-21T19:00:21.453182Z","iopub.execute_input":"2022-09-21T19:00:21.453487Z","iopub.status.idle":"2022-09-21T19:00:28.628137Z","shell.execute_reply.started":"2022-09-21T19:00:21.453455Z","shell.execute_reply":"2022-09-21T19:00:28.627135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(truth_labels, pred_labels))","metadata":{"id":"D94-strtxGUy","outputId":"28d34144-c8cf-49e3-c930-897ffb3fcb5d","execution":{"iopub.status.busy":"2022-09-21T19:00:28.629726Z","iopub.execute_input":"2022-09-21T19:00:28.630111Z","iopub.status.idle":"2022-09-21T19:00:28.805025Z","shell.execute_reply.started":"2022-09-21T19:00:28.630074Z","shell.execute_reply":"2022-09-21T19:00:28.803917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion Matrix","metadata":{}},{"cell_type":"code","source":"cm = confusion_matrix(truth_labels, pred_labels)\n\nimport seaborn as sns \ncolors = sns.color_palette(\"coolwarm\", as_cmap=True)\n\nfig, ax= plt.subplots(1, 1, dpi=200)\nsns.heatmap(cm, annot=True, fmt='d', center=0, ax=ax, square=True, cmap=colors, annot_kws={'fontsize': 8,});\n\nax.set_xlabel('Predicted labels');ax.set_ylabel('True labels'); \nax.set_title('Confusion Matrix'); \nax.xaxis.set_ticklabels([]); ax.yaxis.set_ticklabels([]);","metadata":{"id":"aQD0Ueu4vvRT","outputId":"d14d6f71-22ac-41c3-b4c4-b7c20ef6b86f","execution":{"iopub.status.busy":"2022-09-21T19:00:28.806678Z","iopub.execute_input":"2022-09-21T19:00:28.807040Z","iopub.status.idle":"2022-09-21T19:00:29.732831Z","shell.execute_reply.started":"2022-09-21T19:00:28.807005Z","shell.execute_reply":"2022-09-21T19:00:29.731806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PCA Analysis of features","metadata":{}},{"cell_type":"code","source":"X_embedded = TSNE(n_components=2,init=\"pca\",).fit_transform(features)","metadata":{"id":"t92hafY0YfMs","outputId":"1d62f14a-38bf-4268-9279-c138ece79a4d","execution":{"iopub.status.busy":"2022-09-21T19:00:29.734394Z","iopub.execute_input":"2022-09-21T19:00:29.735025Z","iopub.status.idle":"2022-09-21T19:02:18.471585Z","shell.execute_reply.started":"2022-09-21T19:00:29.734975Z","shell.execute_reply":"2022-09-21T19:02:18.470459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.tight_layout()\nfig, ax = plt.subplots(1, 1, dpi=200)\nplt.xticks([], [])\nplt.yticks([], [])\nplt.title(\"t-SNE\")\n\nax.scatter(X_embedded[:,0], X_embedded[:,1], c=truth_labels,)","metadata":{"id":"Mg4rsaTHcvxd","outputId":"3dc700c9-a248-4cf2-bf00-d16b670640a4","execution":{"iopub.status.busy":"2022-09-21T19:02:18.473386Z","iopub.execute_input":"2022-09-21T19:02:18.473824Z","iopub.status.idle":"2022-09-21T19:02:19.204699Z","shell.execute_reply.started":"2022-09-21T19:02:18.473786Z","shell.execute_reply":"2022-09-21T19:02:19.203810Z"},"trusted":true},"execution_count":null,"outputs":[]}]}