{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install py7zr\n\n!pip install wandb\nimport os\n!WANDB_API_KEY=$46c39bbac8b187eeb815386597ca25ba414d4c5e\nos.environ['WANDB_API_KEY'] = '46c39bbac8b187eeb815386597ca25ba414d4c5e'\nimport wandb","metadata":{"execution":{"iopub.status.busy":"2022-09-01T14:56:58.105858Z","iopub.execute_input":"2022-09-01T14:56:58.106331Z","iopub.status.idle":"2022-09-01T14:57:23.821771Z","shell.execute_reply.started":"2022-09-01T14:56:58.106240Z","shell.execute_reply":"2022-09-01T14:57:23.820350Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import classification_report\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport py7zr\nfrom py7zr import unpack_7zarchive\nimport shutil\nimport glob\nimport random as rd\n\nimport torch \nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nimport torchvision.transforms as transforms\n# from torchvision.io import read_image\nimport torch.nn.functional as F\n\nimport pandas as pd\nfrom PIL import Image\nimport numpy as np\nimport sys\nimport os \nfrom pprint import pprint\n","metadata":{"execution":{"iopub.status.busy":"2022-09-01T14:57:27.056906Z","iopub.execute_input":"2022-09-01T14:57:27.057676Z","iopub.status.idle":"2022-09-01T14:57:27.066211Z","shell.execute_reply.started":"2022-09-01T14:57:27.057634Z","shell.execute_reply":"2022-09-01T14:57:27.064435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/data\nshutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)\nshutil.unpack_archive(\"../input/cifar-10/test.7z\", \"/kaggle/working/data/\")\nshutil.unpack_archive(\"../input/cifar-10/train.7z\", \"/kaggle/working/data/\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T14:57:27.069382Z","iopub.execute_input":"2022-09-01T14:57:27.070084Z","iopub.status.idle":"2022-09-01T15:12:05.988545Z","shell.execute_reply.started":"2022-09-01T14:57:27.070043Z","shell.execute_reply":"2022-09-01T15:12:05.987391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/data\ntrain_csv = pd.read_csv('../input/cifar-10/trainLabels.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T15:12:05.990303Z","iopub.execute_input":"2022-09-01T15:12:05.990682Z","iopub.status.idle":"2022-09-01T15:12:07.058319Z","shell.execute_reply.started":"2022-09-01T15:12:05.990646Z","shell.execute_reply":"2022-09-01T15:12:07.056805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv","metadata":{"execution":{"iopub.status.busy":"2022-09-01T15:12:07.060475Z","iopub.execute_input":"2022-09-01T15:12:07.060857Z","iopub.status.idle":"2022-09-01T15:12:07.091378Z","shell.execute_reply.started":"2022-09-01T15:12:07.060814Z","shell.execute_reply":"2022-09-01T15:12:07.090392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Dataloader","metadata":{}},{"cell_type":"code","source":"transform = transforms.Compose({\n        transforms.ToTensor(),\n        transforms.Normalize((0.5,0.5,0.5), (0.5, 0.5, 0.5)),\n    })\n\nclass dataset(Dataset):\n\n    classes = {'truck':0, 'airplane':1, 'automobile':2, 'bird':3, 'cat':4,\n           'deer':5, 'dog':6, 'frog':7, 'horse':8, 'ship':9, }\n        \n    def __init__(self, train_csv):\n        super(dataset, self).__init__()\n        self.df = pd.read_csv(train_csv)\n        self.label = self.df['label'].tolist()\n        self.id = self.df['id'].tolist()\n        self.classes = {'truck':0, 'airplane':1, 'automobile':2, 'bird':3, 'cat':4,\n           'deer':5, 'dog':6, 'frog':7, 'horse':8, 'ship':9, }\n       \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        root_dir = '/kaggle/working/data/train/'\n        id_ = self.id[idx]\n        image = os.path.join(root_dir, f'{id_}.png')\n        image = np.array(Image.open(image))   # np.array()\n        label = self.label[idx]\n        label = self.classes[label]\n        \n        image = transform(image)\n        \n        return image, label\n        \n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-01T15:23:11.491545Z","iopub.execute_input":"2022-09-01T15:23:11.492140Z","iopub.status.idle":"2022-09-01T15:23:11.502515Z","shell.execute_reply.started":"2022-09-01T15:23:11.492103Z","shell.execute_reply":"2022-09-01T15:23:11.501200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = dataset(\"../input/cifar-10/trainLabels.csv\")\ntrain_dataset, val_dataset, = torch.utils.data.random_split(data, [45000, 5000], generator=torch.Generator().manual_seed(69))\n# dataloader = DataLoader\ntrain_loader = DataLoader(train_dataset, batch_size = 32)\nval_loader = DataLoader(val_dataset, batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T15:23:12.096063Z","iopub.execute_input":"2022-09-01T15:23:12.096446Z","iopub.status.idle":"2022-09-01T15:23:12.126537Z","shell.execute_reply.started":"2022-09-01T15:23:12.096390Z","shell.execute_reply":"2022-09-01T15:23:12.125572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train CNN model\n","metadata":{}},{"cell_type":"code","source":"class CNN(nn.Module):\n  def __init__(self):\n    super(CNN,self).__init__()\n    self.conv1 = nn.Conv2d(3, 64, 3)\n#     self.pool  = nn.MaxPool2d(2,2)\n    self.pool  = nn.MaxPool2d(4,4)\n    self.norm  = nn.BatchNorm2d(64)\n    self.conv2 = nn.Conv2d(64, 128, 3)\n    self.fc1   = nn.Linear(128*28*28, 256)\n    self.drop  = nn.Dropout(0.3)\n    self.fc2   = nn.Linear(256, 128)\n    self.fc3   = nn.Linear(128,10)\n\n  def forward(self, x):\n    x = (self.conv1(x))\n    print(\"conv1 \", x.size())\n    x = self.pool(x)\n    print(\"pool \", x.size())\n    x = F.relu(self.norm(x))\n    print(\"norm \", x.size())\n    x = F.relu(self.conv2(x))\n    print(\"conv2 \",x.size())\n    x = self.pool(x)\n    print(\"pool \",x.size())\n    x = torch.flatten(x,1)\n    print(\"flatten \", x.size())\n    x = F.relu(self.fc1(x))\n    x = self.drop(x)\n    print(x.size())\n    x = F.relu(self.fc2(x))\n    print(x.size())\n    x = self.fc3(x)\n    print(x.size())\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-09-01T14:41:27.972094Z","iopub.execute_input":"2022-09-01T14:41:27.972593Z","iopub.status.idle":"2022-09-01T14:41:27.989998Z","shell.execute_reply.started":"2022-09-01T14:41:27.972554Z","shell.execute_reply":"2022-09-01T14:41:27.988745Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(device)\nmodel = CNN()\nmodel.to(device)\nhistory = []\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)\nloss_fn = nn.CrossEntropyLoss()\nepochs = 20\ntrain_losses = []\ntrain_accuracy = []\n\nfor epoch in range(epochs):\n    print(f\"epoch {epoch}\")\n    for i, batch in enumerate(train_loader, 0):\n        inputs, labels = batch\n        inputs, labels = inputs.to(device), labels.to(device)\n        # forward pass\n        outputs = model(inputs)\n        loss = loss_fn(outputs, labels)\n        acc = accuracy(outputs, labels)\n        # set gradients to zero\n        optimizer.zero_grad()\n        # backward pass\n        loss.backward()\n        # update gradients\n        optimizer.step()\n        \n        \n        train_losses.append(loss)\n        train_accuracy.append(acc)\n        if(i%100==0):\n            print(f\"\\tloss: {loss} \\tacc: {acc}\")\n\n    result = {}\n    result['train_loss'] = torch.stack(train_losses).mean().item()\n    result[\"train_accuracy\"] = torch.stack(train_accuracy).mean().item()\n    pprint(result)\n    \n    history.append(result)\n","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":true},"execution":{"iopub.status.busy":"2022-09-01T14:41:28.605898Z","iopub.execute_input":"2022-09-01T14:41:28.607232Z","iopub.status.idle":"2022-09-01T14:41:29.183588Z","shell.execute_reply.started":"2022-09-01T14:41:28.607189Z","shell.execute_reply":"2022-09-01T14:41:29.181944Z"},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history","metadata":{"execution":{"iopub.status.busy":"2022-09-01T14:41:29.184390Z","iopub.status.idle":"2022-09-01T14:41:29.184901Z","shell.execute_reply.started":"2022-09-01T14:41:29.184679Z","shell.execute_reply":"2022-09-01T14:41:29.184701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Currently the model is overfitting on train data. so we'll try and intoduce a validation set during training.","metadata":{}},{"cell_type":"code","source":"def accuracy(outputs, labels):\n    _, pred = torch.max(outputs, dim=1)\n    return torch.tensor(torch.sum(pred==labels)/len(pred))\n    \nclass imageClassification(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        pass\n    \n    def training_step(self, batch):\n        inputs, labels = batch\n        inputs, labels = inputs.to(device), labels.to(device)\n        outputs = self(inputs)\n        loss_func = nn.CrossEntropyLoss()\n        loss = loss_func(outputs, labels)\n        acc = accuracy(outputs, labels)\n        return loss, acc\n    \n    def validation_step(self, batch):\n        inputs, labels = batch\n        inputs, labels = inputs.to(device), labels.to(device)\n        outputs = self(inputs)\n        loss_func = nn.CrossEntropyLoss()\n        loss = loss_func(outputs, labels)\n        acc = accuracy(outputs, labels)\n        return {'loss': loss, \"acc\": acc}\n    \n    def validation_epoch(self, result):\n        acc = [x['acc'] for x in result]\n        losses = [x['loss'] for x in result]\n        val_loss = torch.stack(losses).mean()\n        val_acc = torch.stack(acc).mean()        \n        return {'val_loss':val_loss, 'val_acc':val_acc}\n    \n    def end_of_epoch(self, result):\n        print(f\"train_accuracy: {result['train_acc']*100:.2f}  train_loss: {result['train_loss']*100:.2f}\")\n        print(f\"val_accuracy:   {result['val_acc']*100:.2f}    val_loss: {result['val_loss']*100:.2f}\")\n        print('-----------------------------------------------------------------')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T15:23:14.915994Z","iopub.execute_input":"2022-09-01T15:23:14.916352Z","iopub.status.idle":"2022-09-01T15:23:14.929898Z","shell.execute_reply.started":"2022-09-01T15:23:14.916319Z","shell.execute_reply":"2022-09-01T15:23:14.927475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN(imageClassification):\n    def __init__(self):\n        super(CNN, self).__init__()\n        self.network = nn.Sequential(\n            nn.Conv2d(3, 64, 3),\n#             nn.nn.AvgPool2d(2),\n            nn.MaxPool2d(2,2),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.Conv2d(64, 128, 3),\n            nn.MaxPool2d(2,2),\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            nn.Flatten(end_dim=-1),\n            nn.Linear(128*6*6, 256),\n            nn.Dropout(0.4),\n            nn.ReLU(),            \n            nn.Linear(256, 128),\n            nn.Dropout(0.5),\n            nn.ReLU(),\n            nn.Linear(128,10)\n        )\n        \n    def forward(self, x):\n        return self.network(x)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T15:23:15.359729Z","iopub.execute_input":"2022-09-01T15:23:15.360379Z","iopub.status.idle":"2022-09-01T15:23:15.382838Z","shell.execute_reply.started":"2022-09-01T15:23:15.360335Z","shell.execute_reply":"2022-09-01T15:23:15.374156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef evaluate(model, dataloader):\n    model.eval()\n    result = [model.validation_step(batch) for batch in dataloader]\n    return model.validation_epoch(result)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-01T15:23:16.185903Z","iopub.execute_input":"2022-09-01T15:23:16.186270Z","iopub.status.idle":"2022-09-01T15:23:16.191887Z","shell.execute_reply.started":"2022-09-01T15:23:16.186236Z","shell.execute_reply":"2022-09-01T15:23:16.190830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def log_results_wandb(result):\n    wandb.log({\"train loss\":result['train_loss'],\n               \"train acc\": result['train_acc'],\n               \"val acc\": result['val_acc'],\n               \"val loss\": result[\"val_loss\"]})","metadata":{"execution":{"iopub.status.busy":"2022-09-01T15:23:17.281291Z","iopub.execute_input":"2022-09-01T15:23:17.281939Z","iopub.status.idle":"2022-09-01T15:23:17.297717Z","shell.execute_reply.started":"2022-09-01T15:23:17.281879Z","shell.execute_reply":"2022-09-01T15:23:17.294669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, train_loader, val_loader, epochs=50, learning_rate=0.001):\n    history = []\n#     optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n    model.train()\n    for _, epoch in enumerate(range(epochs)):\n        print(f\"Epoch {_}: \")\n        train_losses = []\n        train_acc = []\n        \n        for i, batch in enumerate(train_loader, 0):\n            loss, acc = model.training_step(batch)\n            loss.backward()    \n            optimizer.step()\n            optimizer.zero_grad()\n            \n            if(i%100==0):\n                print(f\"\\tacc: {acc} \\tloss: {loss}\")\n            \n            train_losses.append(loss)\n            train_acc.append(acc)\n            \n        result = evaluate(model, val_loader)\n        result[\"train_loss\"] = torch.stack(train_losses).mean()\n        result[\"train_acc\"] = torch.stack(train_acc).mean()\n        \n        log_results_wandb(result)\n        \n        history.append(result)\n        model.end_of_epoch(result)\n        \n    return model, history\n        ","metadata":{"execution":{"iopub.status.busy":"2022-09-01T16:49:18.162304Z","iopub.execute_input":"2022-09-01T16:49:18.163078Z","iopub.status.idle":"2022-09-01T16:49:18.171751Z","shell.execute_reply.started":"2022-09-01T16:49:18.163039Z","shell.execute_reply":"2022-09-01T16:49:18.170737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    'model':'CNN (2 layers)',\n    'architecture': '2*CPoolBnormR-Flat-LDrop-LDropR-Softmax',\n    'epochs': 50,\n    'batch_size': 32,\n    'optimizer': 'Adam',\n    'learning_rate': 0.00001,\n    'CNN1': 64,\n    'CNN2': 128,\n    'Pool': ['max', [2, 2]],\n    'BatchNorm': True,\n    'dropout': [0.5, 0.5],\n    'dense1_units': 128*2*2,\n    'dense2_units': 256,\n}","metadata":{"execution":{"iopub.status.busy":"2022-09-01T17:00:18.324833Z","iopub.execute_input":"2022-09-01T17:00:18.325818Z","iopub.status.idle":"2022-09-01T17:00:18.331562Z","shell.execute_reply.started":"2022-09-01T17:00:18.325779Z","shell.execute_reply":"2022-09-01T17:00:18.330584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNN()\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nmodel.to(device)\n\nrun = wandb.init(config=config, project='cifar', group='CNN model', notes='2 Conv layer cnn model')\nwandb.watch(model) \n\nmodel, history = train(model, train_loader, val_loader, 50, 0.00001)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T17:00:19.463105Z","iopub.execute_input":"2022-09-01T17:00:19.464134Z","iopub.status.idle":"2022-09-01T17:23:18.338965Z","shell.execute_reply.started":"2022-09-01T17:00:19.464083Z","shell.execute_reply":"2022-09-01T17:23:18.337624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T17:24:55.495761Z","iopub.execute_input":"2022-09-01T17:24:55.496180Z","iopub.status.idle":"2022-09-01T17:24:59.655431Z","shell.execute_reply.started":"2022-09-01T17:24:55.496144Z","shell.execute_reply":"2022-09-01T17:24:59.654577Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Architecture Performance logging (manual) \nafter 10 epochs: <br>\nlr: 0.01\n1. CPBR-CPR-F-LDrR-LR-Sm \n    - train_accuracy: 94.60    train_loss: 16.06\n    - val_accuracy:   68.97    val_loss: 162.68 \n    - high overfiting\n2. CBR-CR-F-LDrR-LR-Sm\n    - train_accuracy: 98.34  train_loss: 5.42\n    - val_accuracy:   62.62    val_loss: 244.06\n    - even higher overfitting\n3. CBR-CBR-F-LDrR-LR-Sm\n    - train_accuracy: 94.65  train_loss: 17.22\n    - val_accuracy:   61.25    val_loss: 191.07\n   \n4. CPBR-CPBR-F-LDrR-LDrR-Sm\n    - train_accuracy: 76.00  train_loss: 68.23\n    - val_accuracy:   64.55    val_loss: 118.88\n5. CPBR-CPBR-F-LDrR-LDrR-Sm (4 and 2 pools)\n    - train_accuracy: 82.60  train_loss: 48.68\n    - val_accuracy:   65.57    val_loss: 129.00","metadata":{}},{"cell_type":"markdown","source":"### Further analysis done in wandb: <a href=\"https://wandb.ai/mindadeepam/cifar?workspace=user-mindadeepam\">link</a>","metadata":{}}]}