{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nfrom torch import nn \nfrom torchvision import transforms, datasets\nfrom matplotlib import pyplot as plt \nimport numpy as np \nimport torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:53:55.723328Z","iopub.execute_input":"2022-10-06T05:53:55.723820Z","iopub.status.idle":"2022-10-06T05:53:57.702047Z","shell.execute_reply.started":"2022-10-06T05:53:55.723715Z","shell.execute_reply":"2022-10-06T05:53:57.700852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:53:57.704287Z","iopub.execute_input":"2022-10-06T05:53:57.704815Z","iopub.status.idle":"2022-10-06T05:53:57.709161Z","shell.execute_reply.started":"2022-10-06T05:53:57.704765Z","shell.execute_reply":"2022-10-06T05:53:57.708235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose(\n    [transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n\nbatch_size = 16\n\ntrainset = datasets.CIFAR10(root='./data', train=True,\n                                        download=True, transform=transform)\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n                                          shuffle=True, num_workers=2)\n\ntestset = datasets.CIFAR10(root='./data', train=False,\n                                       download=True, transform=transform)\ntestloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n                                         shuffle=False, num_workers=2)\n\nclasses = ('plane', 'car', 'bird', 'cat',\n           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:53:57.710868Z","iopub.execute_input":"2022-10-06T05:53:57.711432Z","iopub.status.idle":"2022-10-06T05:54:05.841826Z","shell.execute_reply.started":"2022-10-06T05:53:57.711397Z","shell.execute_reply":"2022-10-06T05:54:05.840948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = next(iter(trainset))\ndef showBatch(sample):\n    \n    img = sample.numpy()\n    plt.imshow(np.transpose(img,(1,2,0)))\nsamplebatch = next(iter(trainloader))\nimages,labels=samplebatch\nshowBatch(torchvision.utils.make_grid(images))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:54:05.844697Z","iopub.execute_input":"2022-10-06T05:54:05.845618Z","iopub.status.idle":"2022-10-06T05:54:06.193132Z","shell.execute_reply.started":"2022-10-06T05:54:05.845582Z","shell.execute_reply":"2022-10-06T05:54:06.191914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SigmoidNet = nn.Sequential(nn.Linear(32*32*3,50),\n                          nn.Sigmoid(),\n                          nn.Linear(50,8),\n                          nn.Sigmoid(),\n                          nn.Linear(8,10))\nTanNet = nn.Sequential(nn.Linear(32*32*3,50),\n                          nn.Tanh(),\n                          nn.Linear(50,8),\n                          nn.Tanh(),\n                          nn.Linear(8,10))\nReluNet = nn.Sequential(nn.Linear(32*32*3,50),\n                          nn.ReLU(),\n                          nn.Linear(50,8),\n                          nn.ReLU(),\n                          nn.Linear(8,10))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:54:06.194974Z","iopub.execute_input":"2022-10-06T05:54:06.195318Z","iopub.status.idle":"2022-10-06T05:54:06.208507Z","shell.execute_reply.started":"2022-10-06T05:54:06.195285Z","shell.execute_reply":"2022-10-06T05:54:06.207792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pytorchsummary","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:54:06.209626Z","iopub.execute_input":"2022-10-06T05:54:06.210124Z","iopub.status.idle":"2022-10-06T05:54:19.043589Z","shell.execute_reply.started":"2022-10-06T05:54:06.210091Z","shell.execute_reply":"2022-10-06T05:54:19.042398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorchsummary import summary\nprint('_'*50,'Sigmid','_'*150)\nsummary((3*32*32,),SigmoidNet)\nprint('_'*50,'TanH','_'*150)\nsummary((3*32*32,),TanNet)\nprint('_'*50,'ReLU','_'*150)\nsummary((3*32*32,),ReluNet)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:54:19.045432Z","iopub.execute_input":"2022-10-06T05:54:19.045809Z","iopub.status.idle":"2022-10-06T05:54:19.083445Z","shell.execute_reply.started":"2022-10-06T05:54:19.045768Z","shell.execute_reply":"2022-10-06T05:54:19.082668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model,criterion,trainloader,valloader,optimizer,epochs=20):\n    acc = {'trainAcc':[], \n                 'valAcc':[]}\n    \n    for e in range(epochs):\n        print(f'Epoch {e+1}/{epochs}')\n        correct=correct_=0\n        model.train()\n\n        for (x,y) in tqdm(trainloader):\n            optimizer.zero_grad()\n            yhat = model(x.view(-1,3*32*32))\n            loss = criterion(yhat,y)\n            loss.backward()\n            optimizer.step()\n            _,label = torch.max(yhat,1)\n            correct +=(label==y).sum().item()\n            \n        acc['trainAcc'].append(correct/(len(trainloader)*batch_size))\n        print(f'Train || Correct/total: {correct}/{len(trainloader)*batch_size}')\n        model.eval()\n        for x_,y_ in tqdm(valloader):\n            yhat_ = model(x_.view(-1,3*32*32))\n            _,label_ = torch.max(yhat_,1)\n            correct_ +=(label==y_).sum().item()\n        acc['valAcc'].append(correct_/(len(valloader)*batch_size))\n        print(f'Validation || Correct/total: {correct_}/{len(valloader)*batch_size}')\n    return acc","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:54:19.084690Z","iopub.execute_input":"2022-10-06T05:54:19.085500Z","iopub.status.idle":"2022-10-06T05:54:19.096146Z","shell.execute_reply.started":"2022-10-06T05:54:19.085466Z","shell.execute_reply":"2022-10-06T05:54:19.094704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:54:19.097946Z","iopub.execute_input":"2022-10-06T05:54:19.098309Z","iopub.status.idle":"2022-10-06T05:54:19.107703Z","shell.execute_reply.started":"2022-10-06T05:54:19.098278Z","shell.execute_reply":"2022-10-06T05:54:19.106706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train for Sigmoid Net","metadata":{}},{"cell_type":"code","source":"optimizer = torch.optim.SGD(SigmoidNet.parameters(),lr=lr)\nsigmoid_results = train(SigmoidNet,criterion,trainloader,testloader,optimizer)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T05:54:19.110365Z","iopub.execute_input":"2022-10-06T05:54:19.110679Z","iopub.status.idle":"2022-10-06T06:00:36.713341Z","shell.execute_reply.started":"2022-10-06T05:54:19.110649Z","shell.execute_reply":"2022-10-06T06:00:36.711933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train for Tan Net","metadata":{}},{"cell_type":"code","source":"optimizer = torch.optim.SGD(TanNet.parameters(),lr=lr)\ntan_results = train(TanNet,criterion,trainloader,testloader,optimizer)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T06:00:36.715416Z","iopub.execute_input":"2022-10-06T06:00:36.715776Z","iopub.status.idle":"2022-10-06T06:06:57.913971Z","shell.execute_reply.started":"2022-10-06T06:00:36.715738Z","shell.execute_reply":"2022-10-06T06:06:57.912888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train for Relu Net\n","metadata":{}},{"cell_type":"code","source":"optimizer = torch.optim.SGD(ReluNet.parameters(),lr=lr)\nrelu_results = train(ReluNet,criterion,trainloader,testloader,optimizer)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T06:06:57.915801Z","iopub.execute_input":"2022-10-06T06:06:57.916169Z","iopub.status.idle":"2022-10-06T06:13:16.956981Z","shell.execute_reply.started":"2022-10-06T06:06:57.916132Z","shell.execute_reply":"2022-10-06T06:13:16.955655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compare the Results","metadata":{}},{"cell_type":"code","source":"plt.plot(sigmoid_results['trainAcc'],label='sigmoid',color='blue')\nplt.plot(tan_results['trainAcc'],label='tanh',color='red')\nplt.plot(relu_results['trainAcc'],label='relu',color='green')\nplt.ylabel('Training Accuracy')\nplt.title('Acc vs epochs')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T06:28:17.085881Z","iopub.execute_input":"2022-10-06T06:28:17.086359Z","iopub.status.idle":"2022-10-06T06:28:17.334578Z","shell.execute_reply.started":"2022-10-06T06:28:17.086318Z","shell.execute_reply":"2022-10-06T06:28:17.333691Z"},"trusted":true},"execution_count":null,"outputs":[]}]}