{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torch.backends.cudnn as cudnn\nimport numpy as np\nimport torchvision\nfrom torchvision import datasets, models, transforms\nimport matplotlib.pyplot as plt\nimport time\nimport os\nfrom PIL import Image\nfrom tempfile import TemporaryDirectory\nfrom torch.utils.data import Dataset\ncudnn.benchmark = True\nimport pandas as pd\nplt.ion()   # interactive mode\nfrom tqdm import tqdm\n\nimport plotly.express as px\n\n!pip install torch-summary\nfrom torchsummary import summary\nimport os\nimport shutil\nimport random\nimport json","metadata":{"_uuid":"b9f2aa39-9291-4c3b-84f8-77188aa0ebef","_cell_guid":"20eaef28-413f-4107-9aa0-10895e118d48","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-06T05:16:15.432089Z","iopub.execute_input":"2023-07-06T05:16:15.432443Z","iopub.status.idle":"2023-07-06T05:16:30.556599Z","shell.execute_reply.started":"2023-07-06T05:16:15.432415Z","shell.execute_reply":"2023-07-06T05:16:30.555079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Steps\n1. Retrain the efficient net model on CIFA-10 Dataset, evaluate it's performance.\n2. Write code to calculate the sparsity of CNN layers of model.\n3. Iteratively prune the networks.\n    -Prune x% and then evaluate model.\n    - Retrain model for n epochs.\n    -Evaluate\n    -Prune x-y % and repeat.\n\nRemove ***%%script echo*** block to run it, I included it so that i don't mess up the code while doing the pruning by runing some block and change everything.\n\nI will be only pruning CNN parameters.","metadata":{}},{"cell_type":"code","source":"#All the necessary code to load and configure the dataset \nbatch_size = 64\npath = '/kaggle/input/cifar10-python'\ndata_dir = '/kaggle/input/tiny-imagenet-reorganized/tiny-imagenet-200'\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n\ndata_transforms = {\n    'train': transforms.Compose([\n        transforms.Resize(224),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize(224),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ]),\n}\n\n\ntrainset = torchvision.datasets.CIFAR10(root=path, \n                                        train=True, download=False, \n                                        transform=data_transforms['train'])\n\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=2)\n\n\ntestset = torchvision.datasets.CIFAR10(root=path, \n                                        train=False, download=False, \n                                        transform=data_transforms['test'])\n\n                                     \ntestloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n                                         shuffle=False, num_workers=2)\n\nclasses = ('plane', 'car', 'bird', 'cat',\n           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\n\n\n#Making things easy, so storing it in a dictionary \ndataset_size = [len(trainset), len(testset)]\ndataloaders= {\n    \"train\": trainloader,\n    \"val\": testloader\n}\ndataset_sizes = {\n    \"train\": dataset_size[0],\n    \"val\": dataset_size[1]\n}\n\nclass_names = trainset.classes","metadata":{"_uuid":"75883057-e4a5-49d3-a756-736587d1ee3d","_cell_guid":"20bd1e1b-bc47-4f1d-bd6d-c63671d1e28a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-06T01:09:36.013962Z","iopub.execute_input":"2023-07-06T01:09:36.014848Z","iopub.status.idle":"2023-07-06T01:09:38.669699Z","shell.execute_reply.started":"2023-07-06T01:09:36.014806Z","shell.execute_reply":"2023-07-06T01:09:38.668190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%script echo  #exclude block from running\ndef imshow(inp, title=None):\n    \"\"\"Display image for Tensor.\"\"\"\n    inp = inp.numpy().transpose((1, 2, 0)) #transposing to h,w,c\n    mean = np.array([0.485, 0.456, 0.406])  \n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean    #back to original\n    inp = np.clip(inp, 0, 1)\n    plt.figure(figsize=(12,10))\n    plt.imshow(inp)\n    plt.title('Sample Images from CIFAR-10 Dataset')\n    if title is not None:\n        plt.title(title)\n    plt.savefig('CIFAR10')\n    plt.pause(0.001)  # pause a bit so that plots are updated","metadata":{"_uuid":"422458d3-e354-4701-b5fd-d49ee21af772","_cell_guid":"399f0ea5-4a97-4862-810c-42c2b3feb0fb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-06T02:26:20.020106Z","iopub.execute_input":"2023-07-06T02:26:20.020503Z","iopub.status.idle":"2023-07-06T02:26:20.028716Z","shell.execute_reply.started":"2023-07-06T02:26:20.020472Z","shell.execute_reply":"2023-07-06T02:26:20.027756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#visualizing trian dataset\n# Get a batch of training data\ninputs, classes = next(iter(trainloader))\n\n# Make a grid from batch\nout = torchvision.utils.make_grid(inputs)\n\nimshow(out)\n\n#visualizing images in validation datatset\nimages,labels = next(iter(testloader))\nout = torchvision.utils.make_grid(images)\nimshow(out, title= 'visualizing images in validation datatset')","metadata":{"_uuid":"8c6071b0-f7b2-4ccf-bb08-359da862070a","_cell_guid":"fdd30996-8b89-4c14-b977-b61fa4009e3c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-06T02:26:39.204813Z","iopub.execute_input":"2023-07-06T02:26:39.205252Z","iopub.status.idle":"2023-07-06T02:26:46.616271Z","shell.execute_reply.started":"2023-07-06T02:26:39.205218Z","shell.execute_reply":"2023-07-06T02:26:46.615142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#Function for training the model, forware prop and backward prop \ndef train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n    since = time.time()\n\n    #storing epoch data\n    epoch_data =     {\n        'epoch': [],\n        'train': {'loss': [], 'acc': []},\n        'val': {'loss': [], 'acc': [] }\n    }\n    \n    # Create a temporary directory in Kaggle's temp directory\n    tempdir = '/kaggle/working/temp'\n    os.makedirs(tempdir, exist_ok=True)\n    best_model_params_path = os.path.join(tempdir, 'best_model_params.pt')\n\n    torch.save(model.state_dict(), best_model_params_path)\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch+1}/{num_epochs}')\n        print('-' * 10)\n        epoch_data['epoch'].append(epoch+1)\n        \n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n            running_loss = 0.0\n            running_corrects = 0\n\n            \n\n            for inputs, labels in tqdm(dataloaders[phase], leave=False):\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                optimizer.zero_grad()\n\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n            epoch_data[phase]['loss'].append(epoch_loss)\n            epoch_data[phase]['acc'].append(epoch_acc)\n\n            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                torch.save(model.state_dict(), best_model_params_path)\n\n        print()\n\n    time_elapsed = time.time() - since\n    print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')\n    print(f'Best val Acc: {best_acc:4f}')\n\n    model.load_state_dict(torch.load(best_model_params_path))\n\n    # Clean up the temporary directory\n    shutil.rmtree(tempdir)\n\n    return model, epoch_data\n\n\n#Evaluating the model performance on the test set\ndef evaluate_model(model, criterion = nn.CrossEntropyLoss() , dataloader = testloader ):\n    model.eval()\n    running_loss = 0.0\n    running_corrects = 0\n\n    for inputs, labels in dataloader:\n        inputs = inputs.to(device)\n        labels = labels.to(device)\n\n        with torch.no_grad():\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n            loss = criterion(outputs, labels)\n\n        running_loss += loss.item() * inputs.size(0)\n        running_corrects += torch.sum(preds == labels.data)\n\n    eval_loss = running_loss / len(dataloader.dataset)\n    eval_acc = running_corrects.double() / len(dataloader.dataset)\n\n    print(f'Evaluation Loss: {eval_loss:.4f} Acc: {eval_acc:.4f}')\n\n    return eval_loss, eval_acc\n\n\n#Code to visualize the predictions of model \n\ndef visualize_model(model, num_images=3):\n    was_training = model.training\n    model.eval()\n    images_so_far = 0\n    fig = plt.figure()\n\n    with torch.no_grad():\n        for i, (inputs, labels) in enumerate(dataloaders['val']):\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n\n            outputs = model(inputs)\n            _, preds = torch.max(outputs, 1)\n\n            for j in range(inputs.size()[0]):\n                images_so_far += 1\n                ax = plt.subplot(num_images//2, 2, images_so_far)\n                ax.axis('off')\n                ax.set_title(f'predicted: {class_names[preds[j]]}')\n                imshow(inputs.cpu().data[j])\n\n                if images_so_far == num_images:\n                    model.train(mode=was_training)\n                    return\n        model.train(mode=was_training)","metadata":{"_uuid":"07718f18-ba1c-432d-9a88-e472a9439e0c","_cell_guid":"49ba131b-1dce-4b41-adcd-b805508c7b60","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-06T01:09:38.671251Z","iopub.execute_input":"2023-07-06T01:09:38.671655Z","iopub.status.idle":"2023-07-06T01:09:38.698212Z","shell.execute_reply.started":"2023-07-06T01:09:38.671600Z","shell.execute_reply":"2023-07-06T01:09:38.697017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reframing the Efficient net V2 (small) for 10 classes","metadata":{}},{"cell_type":"code","source":"%%script echo  #exclude block from running temp\n#Model Definations \nmodel_ft = models.efficientnet_v2_s(weights='IMAGENET1K_V1') \n\n# for param in model_ft.parameters():\n#     param.requires_grad = False\n\nnum_ftrs = model_ft.classifier[1].in_features #get the total number of features input to the FC layer.\n# Here the size of each output sample is set to 2.\n\n# Alternatively, it can be generalized to ``nn.Linear(num_ftrs, len(class_names))``.\nmodel_ft.classifier[1] = nn.Linear(num_ftrs, 10)\n\nmodel_ft = model_ft.to(device)\n\ncriterion = nn.CrossEntropyLoss()\n\n# Observe that all parameters are being optimized\noptimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)","metadata":{"_uuid":"6e84e4a1-bd23-42b8-9877-7dc3ac2ef5aa","_cell_guid":"ce9b974d-d15c-4daa-b294-30f687d287b9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-05T12:50:05.005458Z","iopub.execute_input":"2023-07-05T12:50:05.005909Z","iopub.status.idle":"2023-07-05T12:50:05.027495Z","shell.execute_reply.started":"2023-07-05T12:50:05.005877Z","shell.execute_reply":"2023-07-05T12:50:05.026678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This was done, model was trained for 20 epochs and I was able to rich accuracy of 97.92%. The highest SOTA results on CIFAR-10 using efficientnet-v2(small) was 98.7.","metadata":{}},{"cell_type":"code","source":"%%script echo\n#Training the model \nmodel_ft, epoch_data = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,\n                       num_epochs=10)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.468200Z","iopub.execute_input":"2023-07-03T14:14:40.468551Z","iopub.status.idle":"2023-07-03T14:14:40.480304Z","shell.execute_reply.started":"2023-07-03T14:14:40.468526Z","shell.execute_reply":"2023-07-03T14:14:40.479038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\n#Saving the model to working directory \npath = '/kaggle/working/'\nbest_model = os.path.join(path, 'efficientnet.pt')\ntorch.save(model_ft.state_dict(), best_model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.481885Z","iopub.execute_input":"2023-07-03T14:14:40.482359Z","iopub.status.idle":"2023-07-03T14:14:40.492980Z","shell.execute_reply.started":"2023-07-03T14:14:40.482320Z","shell.execute_reply":"2023-07-03T14:14:40.491863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\n#Saving the train data\nimport json\nn = random.randint(4, 75)\nfilename = f'plot_data{n}.json'\nwith open(filename, 'w') as file:\n    json.dump(plot_data, file)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.494242Z","iopub.execute_input":"2023-07-03T14:14:40.494673Z","iopub.status.idle":"2023-07-03T14:14:40.509121Z","shell.execute_reply.started":"2023-07-03T14:14:40.494643Z","shell.execute_reply":"2023-07-03T14:14:40.508320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\npath = '/kaggle/working/'\nbest_model = os.path.join(path, 'best_model_9791.pt')\ntorch.save(model, best_model)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.514091Z","iopub.execute_input":"2023-07-03T14:14:40.514735Z","iopub.status.idle":"2023-07-03T14:14:40.524630Z","shell.execute_reply.started":"2023-07-03T14:14:40.514704Z","shell.execute_reply":"2023-07-03T14:14:40.523614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\n#save the data for future plot\nfilename = 'epoch_data3.json'\nwith open(filename, 'w') as file:\n    json.dump(epochs, file)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.525745Z","iopub.execute_input":"2023-07-03T14:14:40.526061Z","iopub.status.idle":"2023-07-03T14:14:40.539950Z","shell.execute_reply.started":"2023-07-03T14:14:40.526029Z","shell.execute_reply":"2023-07-03T14:14:40.539105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plots the Model loss and accuracy graphs\ndef plot_model_train(epoch_data, title = ' '):\n    epoch_data= convert_numpy(epoch_data)\n    epochs = epoch_data['epoch']\n    train_loss = epoch_data['train']['loss']\n    val_loss = epoch_data['val']['loss']\n    train_acc = epoch_data['train']['acc']\n    val_acc = epoch_data['val']['acc']\n\n    # Plotting the training and validation loss\n    plt.figure(1)\n    plt.plot(epochs, train_loss, label='Training Loss', color='blue', linestyle='-', marker='o')\n    plt.plot(epochs, val_loss, label='Validation Loss', color='red', linestyle='--', marker='s')\n    plt.xlabel('Epoch', color='black')\n    plt.ylabel('Loss', color='black')\n    plt.title('Training and validation loss'+title, color='black')\n    plt.legend(loc='upper right', facecolor='white', framealpha=1)\n    plt.grid(color='gray', linestyle='--', linewidth=0.5)\n    plt.tick_params(axis='x', colors='black')\n    plt.tick_params(axis='y', colors='black')\n    plt.savefig('plot2.png')\n    # Plotting the training and validation accuracy\n    plt.figure(2)\n    plt.plot(epochs, train_acc, label='Training Accuracy', color='blue', linestyle='-', marker='^')\n    plt.plot(epochs, val_acc, label='Validation Accuracy', color='red', linestyle='--', marker='d')\n    plt.xlabel('Epoch', color='black')\n    plt.ylabel('Accuracy', color='black')\n    plt.title('Training and validation accuracy'+title, color='black')\n    plt.legend(loc='lower right', facecolor='white', framealpha=1)\n    plt.grid(color='gray', linestyle='--', linewidth=0.5)\n    plt.tick_params(axis='x', colors='black')\n    plt.tick_params(axis='y', colors='black')\n    \n    n = random.randint(0,100)\n    plt.savefig(f'plot{n}.png')\n\n    plt.show()\n    \n\n#the epoch data is stored as tensor need to convert to numpy array\n#accuracy is stored as tensor so -> to array\ndef convert_numpy(epoch_data, correction =None):\n    if correction is None:\n        epochs = epoch_data['epoch']\n    else:\n        epochs = correction\n    train_loss = epoch_data['train']['loss']\n    val_loss = epoch_data['val']['loss']\n    train_acc = epoch_data['train']['acc']\n    val_acc = epoch_data['val']['acc']\n    \n    \n    train_acc = list([x.to('cpu').detach().numpy().astype('float32') for x in train_acc])\n    val_acc = list([x.to('cpu').detach().numpy().astype('float32') for x in val_acc])\n    train_acc = np.ravel(train_acc).tolist()\n    val_acc = np.ravel(val_acc).tolist()\n    \n    epoch_dict =     {\n        'epoch': epochs,\n        'train': {'loss': train_loss, 'acc': train_acc},\n        'val': {'loss': val_loss, 'acc': val_acc}\n    }\n    \n    return epoch_dict\n","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:10:28.025295Z","iopub.execute_input":"2023-07-06T01:10:28.025851Z","iopub.status.idle":"2023-07-06T01:10:28.055169Z","shell.execute_reply.started":"2023-07-06T01:10:28.025803Z","shell.execute_reply":"2023-07-06T01:10:28.054059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\n#Not need for now\n#serialy combining all json files:\n# List of JSON file paths \nfile_paths = ['/kaggle/working/epoch_data.json', '/kaggle/working/epoch_data2.json', \n              \n              '/kaggle/working/epoch_data3.json']\n\nplot_data = {\n        'epoch': [],\n        'train': {'loss': [], 'acc': []},\n        'val': {'loss': [], 'acc': []}\n    }\n\n\n# Read and merge JSON data from each file\nfor file_path in file_paths:\n    with open(file_path, 'r') as file:\n        epoch_data = json.load(file)\n        epochs = epoch_data['epoch']\n        train_loss = epoch_data['train']['loss']\n        val_loss = epoch_data['val']['loss']\n        train_acc = epoch_data['train']['acc']\n        val_acc = epoch_data['val']['acc']\n        \n        plot_data['epoch']+= epochs\n        plot_data['train']['loss']+=train_loss\n        plot_data['val']['loss']+=val_loss\n        plot_data['train']['acc']+=train_acc\n        plot_data['val']['acc']+=val_acc\n        \n","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.574255Z","iopub.execute_input":"2023-07-03T14:14:40.576609Z","iopub.status.idle":"2023-07-03T14:14:40.596327Z","shell.execute_reply.started":"2023-07-03T14:14:40.576579Z","shell.execute_reply":"2023-07-03T14:14:40.595482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\nwith open('plot_data.json', 'w') as file:\n    json.dump(plot_data,file)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.600245Z","iopub.execute_input":"2023-07-03T14:14:40.600962Z","iopub.status.idle":"2023-07-03T14:14:40.611716Z","shell.execute_reply.started":"2023-07-03T14:14:40.600931Z","shell.execute_reply":"2023-07-03T14:14:40.610544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\nplot_model_train(plot_data)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.613018Z","iopub.execute_input":"2023-07-03T14:14:40.614131Z","iopub.status.idle":"2023-07-03T14:14:40.624848Z","shell.execute_reply.started":"2023-07-03T14:14:40.614098Z","shell.execute_reply":"2023-07-03T14:14:40.623862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Starting pruning\n","metadata":{}},{"cell_type":"code","source":"#loading saved model from kaggle working direct\n\nmodel= torch.load('/kaggle/working/best_model_9791.pt', map_location=  torch.device(\"cpu\"))\n# model = model.to(device)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.625926Z","iopub.execute_input":"2023-07-03T14:14:40.626232Z","iopub.status.idle":"2023-07-03T14:14:40.770327Z","shell.execute_reply.started":"2023-07-03T14:14:40.626203Z","shell.execute_reply":"2023-07-03T14:14:40.769196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#To load model in gpu\nmodel= torch.load('/kaggle/working/best_model_9791.pt')\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n# model = model.to(device)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:36:12.013389Z","iopub.execute_input":"2023-07-06T02:36:12.014189Z","iopub.status.idle":"2023-07-06T02:36:12.226988Z","shell.execute_reply.started":"2023-07-06T02:36:12.014149Z","shell.execute_reply":"2023-07-06T02:36:12.225925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parameter distribution before pruning.\nvisualize_par(model, title='Fig: Parameter distribution in EfficientNet-V2 before pruning')","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:37:44.030272Z","iopub.execute_input":"2023-07-06T02:37:44.030705Z","iopub.status.idle":"2023-07-06T02:37:44.154076Z","shell.execute_reply.started":"2023-07-06T02:37:44.030672Z","shell.execute_reply":"2023-07-06T02:37:44.152955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # weight visualization of model\n# import plotly.express as px\n# import matplotlib.cm as cm\n# plt.figure(figsize=(10, 5))\n# plt.title(\"Weights Visualization\")\n# layer_names = []\n\n\n# layer_names = []\n# count =0\n# shape_data=[]\n# for name, param in model.named_parameters():\n#     weights = param.data\n#     if int(weights.dim()) == 4:\n#         if count%10 ==0:\n#             weights = weights.view(weights.shape[0], -1)  # 1D\n#             layer_names.append(name)\n# # #           plt.imshow(weights, cmap='seismic', vmin=np.nanmin(weights), vmax=np.nanmax(weights))\n# #             plt.colorbar()\n# #             plt.title(name)\n# #             plt.axis('off')\n# #             plt.show()\n#             px.strip(weights, x='weight')\n#         count+=1\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:40.993054Z","iopub.execute_input":"2023-07-03T14:14:40.993461Z","iopub.status.idle":"2023-07-03T14:14:40.999123Z","shell.execute_reply.started":"2023-07-03T14:14:40.993426Z","shell.execute_reply":"2023-07-03T14:14:40.997932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameter Calculation","metadata":{}},{"cell_type":"code","source":"#percentage of zero calculation in conv layer only\n\n#Calculate sparsity, works on the mask too but not layer wise\ndef cal_sparsity(model):\n    z=0\n    para= 0\n    for module in filter(lambda m: type(m) == torch.nn.Conv2d, model.modules()):\n        para+= module.weight.numel() \n        z+= torch.sum(module.weight==0)\n    sparsity = z/para\n    print(f'sparisty: {sparsity*100}% \\nZero_Wt: {z}\\nTotal Params: {para}')\n    return sparsity, z, para\n\n\n#Calculates the block wise Sparsity but works only after making pruning permanent\ndef per_zero(model):\n    total_sum=0\n    no_of_zeros=0\n    for name, param in model.named_parameters():\n        weights = param.data\n        if int(weights.dim()) == 4:\n            weights = weights.view(-1)  # 1D\n            total_sum += weights.shape[0]\n            no_of_zeros += torch.sum(weights == 0).item()\n    sparsity = (no_of_zeros/total_sum)*100\n#     print(f'Sparsit:{sparsity}\\n Total Params: {total_sum} \\n Zero Params: {no_of_zeros}' )\n    return sparsity, total_sum, no_of_zeros \n\n\n#weight distribution before the pruning, returns the distriubtion of weights in different blocks\n#efficientnet is abstractly divided in 8 blocks\ndef weight_distribution(model):\n    i = 1\n    MB_conv=[]\n    FMB_con=[]\n    data = []\n    for module in model.features:\n        _, total_par, zero = per_zero(module)\n        a=0\n        b=0\n        temp=[]\n        for md in filter(lambda m: type(m) == torchvision.models.efficientnet.FusedMBConv, module.modules()):\n            a+=1\n            \n        for md in filter(lambda m: type(m) == torchvision.models.efficientnet.MBConv, module.modules()):\n            b+=1\n\n        name = f'Block No. {i}'\n        data.append([i, total_par-zero, a,b ])\n        i+=1\n    data= np.array(data)\n    return data\n\n\n#Visualizing the Parameters\ndef visualize_par(model, title =\"Fig: Visualization of prameters in different Blocks\"):\n    data = weight_distribution(model)\n    df = pd.DataFrame(data, columns=[\"Block\", \"No. of parameters\", \"FusedMBConv\", \"MBConv\"])\n    df[\"FusedMBconv, MBConv\"] = df[\"FusedMBConv\"].astype(str)+','+df[\"MBConv\"].astype(str)\n\n\n    fig = px.bar(df, x=\"Block\",\n                 title=title,\n                 y =\"No. of parameters\", color = 'FusedMBconv, MBConv')\n\n    fig.update_layout(yaxis_type='log')\n    fig.show()\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:30:16.560112Z","iopub.execute_input":"2023-07-06T02:30:16.560561Z","iopub.status.idle":"2023-07-06T02:30:16.580952Z","shell.execute_reply.started":"2023-07-06T02:30:16.560527Z","shell.execute_reply":"2023-07-06T02:30:16.579850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\ndata = weight_distribution(model)\ndf = visualize_par(data)\nprint(df)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:41.020479Z","iopub.execute_input":"2023-07-03T14:14:41.020813Z","iopub.status.idle":"2023-07-03T14:14:41.038178Z","shell.execute_reply.started":"2023-07-03T14:14:41.020782Z","shell.execute_reply":"2023-07-03T14:14:41.037128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Starting Pruning","metadata":{}},{"cell_type":"code","source":"#all import stat\nimport torch.nn.utils.prune as prune\n","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:12:18.744141Z","iopub.execute_input":"2023-07-06T01:12:18.744535Z","iopub.status.idle":"2023-07-06T01:12:18.749981Z","shell.execute_reply.started":"2023-07-06T01:12:18.744502Z","shell.execute_reply":"2023-07-06T01:12:18.748813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#all funcs\n\n#Pruning process\ndef pruning(model, amt=0.4, method= prune.L1Unstructured):\n    parameters_to_prune = [\n        (module, \"weight\") for module in filter(lambda m: type(m) == torch.nn.Conv2d, model.modules())\n    ]\n\n\n    prune.global_unstructured(\n        parameters_to_prune,\n        pruning_method=method,\n        amount=amt,\n    )\n    \n    print('Completed Pruning.........')\n    return model\n\n#Removes the mask and applies pruning permanently \ndef remove_mask(model):\n    for module in filter(lambda m: type(m) == torch.nn.Conv2d, model.modules()):\n        prune.remove(module, 'weight')\n    \n    \n\n#retraining for n epochs\ndef retrain(model, n):\n    criterion = nn.CrossEntropyLoss()\n    optimizer_ft = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n\n    # Decay LR by a factor of 0.1 every 7 epochs\n    exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)\n\n    model, epoch_data = train_model(model, criterion, optimizer_ft, exp_lr_scheduler,\n                           num_epochs=n)\n    \n    return model, epoch_data\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:12:20.140598Z","iopub.execute_input":"2023-07-06T01:12:20.141349Z","iopub.status.idle":"2023-07-06T01:12:20.152301Z","shell.execute_reply.started":"2023-07-06T01:12:20.141313Z","shell.execute_reply":"2023-07-06T01:12:20.151126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Randomly pruning 40% of weights \nmodel = pruning(model)\nloss, acc = evaluate_model(model)\nprint(loss, acc)\n\ns, z , total = cal_sparsity(model)\nprint(s, z , total )","metadata":{"execution":{"iopub.status.busy":"2023-07-03T14:14:41.063634Z","iopub.execute_input":"2023-07-03T14:14:41.064253Z","iopub.status.idle":"2023-07-03T14:15:13.375929Z","shell.execute_reply.started":"2023-07-03T14:14:41.064221Z","shell.execute_reply":"2023-07-03T14:15:13.374554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I retrained the model after pruning it 40%, the accuracy intialy dropped to 92.29%, on retraining, the accuracy increased to 97.52.\n\n#The original model trained on CIFAR-10 for 20 epochs had** 97.92% accuracy.**","metadata":{}},{"cell_type":"code","source":"#retraining for 3 epochs\nmodel, epochs = retrain(model, 3)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Loading the 40% sparse model and evaluating it's performance.\nmodel = torch.load('/kaggle/working/model_40per_sparse.pt')\nloss, acc = evaluate_model(model)\ns, z , total = cal_sparsity(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:01:34.353169Z","iopub.execute_input":"2023-07-05T13:01:34.354078Z","iopub.status.idle":"2023-07-05T13:02:03.049833Z","shell.execute_reply.started":"2023-07-05T13:01:34.354040Z","shell.execute_reply":"2023-07-05T13:02:03.048621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Iteravile pruning: Pruning 30% more parms from the sparse model\n","metadata":{}},{"cell_type":"code","source":"#Iteratively pruning \n#Randomly pruning 40% of weights \nmodel = pruning(model, amt = 0.3)\nloss, acc = evaluate_model(model)\ns, z , total = cal_sparsity(model)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:03:48.763703Z","iopub.execute_input":"2023-07-05T13:03:48.764622Z","iopub.status.idle":"2023-07-05T13:04:11.594051Z","shell.execute_reply.started":"2023-07-05T13:03:48.764570Z","shell.execute_reply":"2023-07-05T13:04:11.592793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#retrain for next 5 epochs \nmodel, epochs = retrain(model, 5)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:04:52.583269Z","iopub.execute_input":"2023-07-05T13:04:52.584382Z","iopub.status.idle":"2023-07-05T13:33:43.648129Z","shell.execute_reply.started":"2023-07-05T13:04:52.584337Z","shell.execute_reply":"2023-07-05T13:33:43.646999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model, '58%_sparse_model.pt')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:33:43.671151Z","iopub.execute_input":"2023-07-05T13:33:43.671552Z","iopub.status.idle":"2023-07-05T13:33:44.486341Z","shell.execute_reply.started":"2023-07-05T13:33:43.671502Z","shell.execute_reply":"2023-07-05T13:33:44.485340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_train(epochs, title= ' on retraining 58% sparse model')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:40:57.187291Z","iopub.execute_input":"2023-07-05T13:40:57.187670Z","iopub.status.idle":"2023-07-05T13:40:58.054305Z","shell.execute_reply.started":"2023-07-05T13:40:57.187638Z","shell.execute_reply":"2023-07-05T13:40:58.053420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss, acc = evaluate_model(model)\ns, z , total = cal_sparsity(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:33:45.519871Z","iopub.execute_input":"2023-07-05T13:33:45.520190Z","iopub.status.idle":"2023-07-05T13:34:08.262331Z","shell.execute_reply.started":"2023-07-05T13:33:45.520158Z","shell.execute_reply":"2023-07-05T13:34:08.260934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pruning 30% more weights","metadata":{}},{"cell_type":"code","source":"model = pruning(model, amt = 0.3)\nloss, acc = evaluate_model(model)\ns, z , total = cal_sparsity(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:36:56.234708Z","iopub.execute_input":"2023-07-05T13:36:56.235069Z","iopub.status.idle":"2023-07-05T13:37:18.740858Z","shell.execute_reply.started":"2023-07-05T13:36:56.235038Z","shell.execute_reply":"2023-07-05T13:37:18.739154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**** The accuracy decreased to 70%, re-training it for 10 epochs.****","metadata":{}},{"cell_type":"code","source":"model, epochs = retrain(model, 10)\ntorch.save(model, '70%_sparse_model.pt')\nplot_model_train(epochs, title= ' on retraining 70% sparse model')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T13:43:33.156747Z","iopub.execute_input":"2023-07-05T13:43:33.157105Z","iopub.status.idle":"2023-07-05T14:41:10.262505Z","shell.execute_reply.started":"2023-07-05T13:43:33.157072Z","shell.execute_reply":"2023-07-05T14:41:10.261435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"label wrong in above plot, it's plot of retraining on 70% sparse ","metadata":{}},{"cell_type":"code","source":"loss, acc = evaluate_model(model)\ns, z , total = cal_sparsity(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T14:41:11.343846Z","iopub.execute_input":"2023-07-05T14:41:11.344174Z","iopub.status.idle":"2023-07-05T14:41:34.018148Z","shell.execute_reply.started":"2023-07-05T14:41:11.344140Z","shell.execute_reply":"2023-07-05T14:41:34.016289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_train(epochs, title= ' on retraining 70% sparse model')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:08:58.469062Z","iopub.execute_input":"2023-07-05T15:08:58.469426Z","iopub.status.idle":"2023-07-05T15:08:59.258368Z","shell.execute_reply.started":"2023-07-05T15:08:58.469392Z","shell.execute_reply":"2023-07-05T15:08:59.257490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try pruning the model again and see the results","metadata":{}},{"cell_type":"code","source":"model = pruning(model, amt=0.2)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:11:08.620245Z","iopub.execute_input":"2023-07-05T15:11:08.620613Z","iopub.status.idle":"2023-07-05T15:11:08.667942Z","shell.execute_reply.started":"2023-07-05T15:11:08.620583Z","shell.execute_reply":"2023-07-05T15:11:08.666920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_model(model)\ncal_sparsity(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:18:23.628737Z","iopub.execute_input":"2023-07-05T15:18:23.629603Z","iopub.status.idle":"2023-07-05T15:18:45.314769Z","shell.execute_reply.started":"2023-07-05T15:18:23.629552Z","shell.execute_reply":"2023-07-05T15:18:45.313590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try retaining and see if the can recover accuracy or not","metadata":{}},{"cell_type":"code","source":"model, epochs = retrain(model, 5)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:19:00.874812Z","iopub.execute_input":"2023-07-05T15:19:00.875479Z","iopub.status.idle":"2023-07-05T15:47:44.574583Z","shell.execute_reply.started":"2023-07-05T15:19:00.875442Z","shell.execute_reply":"2023-07-05T15:47:44.573354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model, '76%_sparse_model.pt')\nplot_model_train(epochs, title= ' on retraining 76% sparse model')","metadata":{"execution":{"iopub.status.busy":"2023-07-05T15:50:02.145562Z","iopub.execute_input":"2023-07-05T15:50:02.146822Z","iopub.status.idle":"2023-07-05T15:50:04.112273Z","shell.execute_reply.started":"2023-07-05T15:50:02.146781Z","shell.execute_reply":"2023-07-05T15:50:04.111439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try pruning more ","metadata":{}},{"cell_type":"code","source":"model = torch.load('/kaggle/working/76%_sparse_model.pt')","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:11:50.503803Z","iopub.execute_input":"2023-07-06T01:11:50.504202Z","iopub.status.idle":"2023-07-06T01:11:54.932671Z","shell.execute_reply.started":"2023-07-06T01:11:50.504173Z","shell.execute_reply":"2023-07-06T01:11:54.931472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We try pruning 20% of remaining weight.","metadata":{}},{"cell_type":"code","source":"model = pruning(model, amt=0.2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_model(model)\ncal_sparsity(model)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:13:38.397899Z","iopub.execute_input":"2023-07-06T01:13:38.398294Z","iopub.status.idle":"2023-07-06T01:14:02.133032Z","shell.execute_reply.started":"2023-07-06T01:13:38.398263Z","shell.execute_reply":"2023-07-06T01:14:02.131613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try retraining it ","metadata":{}},{"cell_type":"code","source":"model, epochs = retrain(model, 5)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:16:15.208204Z","iopub.execute_input":"2023-07-06T01:16:15.208679Z","iopub.status.idle":"2023-07-06T01:45:31.114893Z","shell.execute_reply.started":"2023-07-06T01:16:15.208611Z","shell.execute_reply":"2023-07-06T01:45:31.113655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_train(epochs, title= ' on retraining 81% sparse model')","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:55:00.871267Z","iopub.execute_input":"2023-07-06T01:55:00.872303Z","iopub.status.idle":"2023-07-06T01:55:01.883507Z","shell.execute_reply.started":"2023-07-06T01:55:00.872266Z","shell.execute_reply":"2023-07-06T01:55:01.882529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Need to try few more epochs:\n\nmodel, epochs2 = retrain(model, 5)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:55:28.204953Z","iopub.execute_input":"2023-07-06T01:55:28.205413Z","iopub.status.idle":"2023-07-06T02:24:37.333111Z","shell.execute_reply.started":"2023-07-06T01:55:28.205379Z","shell.execute_reply":"2023-07-06T02:24:37.331477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model, '81%_sparse_model.pt')\nplot_model_train(epochs, title= ' on retraining 81% sparse model')","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:25:00.786569Z","iopub.execute_input":"2023-07-06T02:25:00.787445Z","iopub.status.idle":"2023-07-06T02:25:03.385745Z","shell.execute_reply.started":"2023-07-06T02:25:00.787394Z","shell.execute_reply":"2023-07-06T02:25:03.381257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Let's visualize the model, for that I will have to remove the mask, i.e make the pruning permannet.**","metadata":{}},{"cell_type":"code","source":"#remove mask\nremove_mask(model)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Parameter visualization\nvisualize_par(model, title =\"Fig: Prameter distribution after pruning 81% of the Network \")","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:38:52.887952Z","iopub.execute_input":"2023-07-06T02:38:52.888362Z","iopub.status.idle":"2023-07-06T02:38:53.014819Z","shell.execute_reply.started":"2023-07-06T02:38:52.888330Z","shell.execute_reply":"2023-07-06T02:38:53.013700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model, '81%_pruned_unmasked_model.pt')","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:34:17.354713Z","iopub.execute_input":"2023-07-06T02:34:17.355401Z","iopub.status.idle":"2023-07-06T02:34:17.630597Z","shell.execute_reply.started":"2023-07-06T02:34:17.355366Z","shell.execute_reply":"2023-07-06T02:34:17.629436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo\n#data unstructured\n\naccu_before_pruning : [_, 97.92, 97.53, 97.41,97.50, 97.48,  ]\nprune_per:   [0 ,40,30,30,20,20]\ntotal sparsity: [0, 40, 58, 70,76, 80]\nAccuracy after pruning: [_ ,92.29, 80.37, 63.54, 52.81, 60,]\nretrain epoch:[0, 3, 5, 10, 5, 5, 10]\naccuracy after retrain: [_, 97.53, 97.41,97.50, 97.48, 97.6]\n    ","metadata":{},"execution_count":null,"outputs":[]}]}