{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nfrom torchvision import datasets,transforms\nfrom torch import nn\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport wandb\nimport random\nimport tensorflow as tf","metadata":{"_uuid":"9aefc49a-0036-4839-965b-51c6323e4c87","_cell_guid":"f9650e81-60ab-43b4-b622-f1ac98667629","collapsed":false,"id":"JukxQHA5Ezs3","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:20:26.795272Z","iopub.execute_input":"2021-11-23T20:20:26.796132Z","iopub.status.idle":"2021-11-23T20:20:33.792072Z","shell.execute_reply.started":"2021-11-23T20:20:26.796008Z","shell.execute_reply":"2021-11-23T20:20:33.791102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key = open(\"../input/keyfile/key.txt\").read()","metadata":{"execution":{"iopub.status.busy":"2021-11-23T20:20:33.793939Z","iopub.execute_input":"2021-11-23T20:20:33.794161Z","iopub.status.idle":"2021-11-23T20:20:33.803829Z","shell.execute_reply.started":"2021-11-23T20:20:33.794133Z","shell.execute_reply":"2021-11-23T20:20:33.803019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Narzędzia służące do monitoringu\n- WandB - Chmurowe roziwązanie do monitorowania eksperymentów oraz optymalizacji parametrów lokalnie, dla pojedyńczej osoby lub zespołów (open source)\n- Neptune - Chmurowe rozwiązanie do monitorowania eksperymentów dla jednej osoby w wersji darmowej\n- MLflow - Lokalne rozwiązanie do monitorowania eksperymentów oraz deploymentu modelu\n","metadata":{"_uuid":"0d2752ae-962f-4640-9b5f-49c06d272eaf","_cell_guid":"6c6c9322-21af-47b2-954f-98c5d9b32c82","trusted":true}},{"cell_type":"code","source":"wandb.login(key=key)","metadata":{"_uuid":"32d5cb38-e61a-47f8-905c-0fb31ce310d4","_cell_guid":"6d0a6e0f-f132-4db7-b400-83ca38eb7ce9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:21:27.975570Z","iopub.execute_input":"2021-11-23T20:21:27.976116Z","iopub.status.idle":"2021-11-23T20:21:28.258739Z","shell.execute_reply.started":"2021-11-23T20:21:27.976080Z","shell.execute_reply":"2021-11-23T20:21:28.257886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"_uuid":"864e91e2-bdb7-4f7c-99d7-af53782370e1","_cell_guid":"a2f0cdfa-4184-48a4-b4e3-c9dedf735bcf","collapsed":false,"id":"11C_r_YRFUGa","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:23:09.187864Z","iopub.execute_input":"2021-11-23T20:23:09.188167Z","iopub.status.idle":"2021-11-23T20:23:09.192927Z","shell.execute_reply.started":"2021-11-23T20:23:09.188132Z","shell.execute_reply":"2021-11-23T20:23:09.192277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    'lr': 0.0025,\n    'epochs': 100,\n    'dropout': True\n}","metadata":{"_uuid":"65f3d827-0b8d-4286-9e6a-18ac409f2e46","_cell_guid":"28bfb26d-3a75-496a-8157-0912ffe867a0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:26:37.234811Z","iopub.execute_input":"2021-11-23T20:26:37.235913Z","iopub.status.idle":"2021-11-23T20:26:37.243320Z","shell.execute_reply.started":"2021-11-23T20:26:37.235816Z","shell.execute_reply":"2021-11-23T20:26:37.242438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project=\"tutorial\", entity=\"oodds\", job_type='test', config=config)","metadata":{"_uuid":"ea9f18ac-6bed-4df1-8656-3996df37c8bb","_cell_guid":"d453293a-7011-4f57-b512-612e0f3a4a47","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:26:37.523381Z","iopub.execute_input":"2021-11-23T20:26:37.524065Z","iopub.status.idle":"2021-11-23T20:26:44.664844Z","shell.execute_reply.started":"2021-11-23T20:26:37.524016Z","shell.execute_reply":"2021-11-23T20:26:44.663803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = transforms.Compose([transforms.Resize((32,32)),  #resises the image so it can be perfect for our model.\n                                      transforms.RandomHorizontalFlip(), # FLips the image w.r.t horizontal axis\n                                      transforms.RandomRotation(10),     #Rotates the image to a specified angel\n                                      transforms.RandomAffine(0, shear=10, scale=(0.8,1.2)), #Performs actions like zooms, change shear angles.\n                                      transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2), # Set the color params\n                                      transforms.ToTensor(), # comvert the image to tensor so that it can work with torch\n                                      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) #Normalize all the images\n                               ])\n \n \n \ntransform = transforms.Compose([transforms.Resize((32,32)),\n                               transforms.ToTensor(),\n                               transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n                               ])\ntraining_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) # Data augmentation is only done on training images\nvalidation_dataset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n \ntraining_loader = torch.utils.data.DataLoader(training_dataset, batch_size=100, shuffle=True) # Batch size of 100 i.e to work with 100 images at a time\nvalidation_loader = torch.utils.data.DataLoader(validation_dataset, batch_size = 100, shuffle=False)","metadata":{"_uuid":"e139beda-ee9a-400d-9ce6-b7fcf8d88fda","_cell_guid":"72d79dc9-d08d-41d1-a16e-bfd27023aad1","collapsed":false,"id":"xJ32chBeFoWd","outputId":"0ad7c337-165a-437b-ca4a-78c89009582e","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:26:44.666714Z","iopub.execute_input":"2021-11-23T20:26:44.666964Z","iopub.status.idle":"2021-11-23T20:26:54.774655Z","shell.execute_reply.started":"2021-11-23T20:26:44.666935Z","shell.execute_reply":"2021-11-23T20:26:54.773680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converting the Input images to plot using plt","metadata":{"_uuid":"2202804c-37c9-4572-a84f-afa9bb4a5f5b","_cell_guid":"60f3ccd3-5076-48fc-bdae-22aa49d12128","id":"JWetPW3AMkF9","trusted":true}},{"cell_type":"code","source":"# We need to convert the images to numpy arrays as tensors are not compatible with matplotlib.\ndef im_convert(tensor):  \n    image = tensor.cpu().clone().detach().numpy() # This process will happen in normal cpu.\n    image = image.transpose(1, 2, 0)\n    image = image * np.array((0.5, 0.5, 0.5)) + np.array((0.5, 0.5, 0.5))\n    image = image.clip(0, 1)\n    return image","metadata":{"_uuid":"1ca501d5-8436-4e24-ad23-0d708eb0abf1","_cell_guid":"d3056bae-b19d-4d29-9c44-1b78319ffe43","collapsed":false,"id":"bTtyjj2OI5KS","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:26:54.776132Z","iopub.execute_input":"2021-11-23T20:26:54.776392Z","iopub.status.idle":"2021-11-23T20:26:54.784161Z","shell.execute_reply.started":"2021-11-23T20:26:54.776364Z","shell.execute_reply":"2021-11-23T20:26:54.782839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Different classes in CIPHAR 10 dataset. \nclasses = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')","metadata":{"_uuid":"641c007c-330a-4d52-985e-1bef8e3a7b10","_cell_guid":"4b23006d-3e2d-4fdb-bddc-0830cd516136","collapsed":false,"id":"eidAN_9OJitC","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:26:54.786166Z","iopub.execute_input":"2021-11-23T20:26:54.786799Z","iopub.status.idle":"2021-11-23T20:26:54.797148Z","shell.execute_reply.started":"2021-11-23T20:26:54.786767Z","shell.execute_reply":"2021-11-23T20:26:54.796340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We iter the batch of images to display\ndataiter = iter(training_loader) # converting our train_dataloader to iterable so that we can iter through it. \nimages, labels = dataiter.next() #going from 1st batch of 100 images to the next batch\nfig = plt.figure(figsize=(25, 4)) \n\n# We plot 20 images from our train_dataset\nfor idx in np.arange(20):\n    ax = fig.add_subplot(2, 10, idx+1, xticks=[], yticks=[]) \n    plt.imshow(im_convert(images[idx])) #converting to numpy array as plt needs it.\n    ax.set_title(classes[labels[idx].item()])","metadata":{"_uuid":"9dbf72f8-d926-49f1-8cfa-32936dc06e07","_cell_guid":"e12572de-94d7-48fb-97b9-e54db6b163b7","collapsed":false,"id":"64Ev5i6FJxxW","outputId":"9356a284-39ba-48e6-da1c-eab1c811fb01","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:26:54.798338Z","iopub.execute_input":"2021-11-23T20:26:54.798665Z","iopub.status.idle":"2021-11-23T20:26:55.968058Z","shell.execute_reply.started":"2021-11-23T20:26:54.798633Z","shell.execute_reply":"2021-11-23T20:26:55.967506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Defining our Model","metadata":{"_uuid":"b22eeaad-6279-469f-8065-19a5f36004c0","_cell_guid":"f9fde1c8-7305-4acb-8b60-0ac70b58e0aa","id":"sTh5PVEfMtn4","trusted":true}},{"cell_type":"code","source":"class LeNet(nn.Module):\n    def __init__(self, dropout):\n        super().__init__()\n        self.conv1 = nn.Conv2d(3, 16, 3, 1, padding=1) # input is color image, hence 3 i/p channels. 16 filters, kernal size is tuned to 3 to avoid overfitting, stride is 1 , padding is 1 extract all edge features.\n        self.conv2 = nn.Conv2d(16, 32, 3, 1, padding=1) # We double the feature maps for every conv layer as in pratice it is really good.\n        self.conv3 = nn.Conv2d(32, 64, 3, 1, padding=1)\n        self.fc1 = nn.Linear(4*4*64, 500) # I/p image size is 32*32, after 3 MaxPooling layers it reduces to 4*4 and 64 because our last conv layer has 64 outputs. Output nodes is 500\n        if dropout: self.dropout1 = nn.Dropout(0.5)\n        else: self.dropout1 = None\n        self.fc2 = nn.Linear(500, 10) # output nodes are 10 because our dataset have 10 different categories\n    def forward(self, x):\n        x = F.relu(self.conv1(x)) #Apply relu to each output of conv layer.\n        x = F.max_pool2d(x, 2, 2) # Max pooling layer with kernal of 2 and stride of 2\n        x = F.relu(self.conv2(x))\n        x = F.max_pool2d(x, 2, 2)\n        x = F.relu(self.conv3(x))\n        x = F.max_pool2d(x, 2, 2)\n        x = x.view(-1, 4*4*64) # flatten our images to 1D to input it to the fully connected layers\n        x = F.relu(self.fc1(x))\n        if self.dropout1:\n            x = self.dropout1(x) # Applying dropout b/t layers which exchange highest parameters. This is a good practice\n        x = self.fc2(x)\n        return x","metadata":{"_uuid":"0fe43c0e-cfd9-4bfb-af2d-4cfafd31ad64","_cell_guid":"06c69033-2f3f-43c4-847b-0a51dee8ba5e","collapsed":false,"id":"jxIIxm1aK2At","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:26:55.969071Z","iopub.execute_input":"2021-11-23T20:26:55.969835Z","iopub.status.idle":"2021-11-23T20:26:55.983616Z","shell.execute_reply.started":"2021-11-23T20:26:55.969782Z","shell.execute_reply":"2021-11-23T20:26:55.982813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LeNet(dropout=wandb.config['dropout']).to(device) # run our model on cuda GPU for faster results\nmodel","metadata":{"_uuid":"99c2761c-5842-4a74-a0ec-f2eda862f3c6","_cell_guid":"3deb5d86-1bcb-4111-b4e6-8c9ac1b71e15","collapsed":false,"id":"Qa1M7e7vOre5","outputId":"4742bd2d-f356-44df-98cc-3ce48e473d1a","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:26:55.984921Z","iopub.execute_input":"2021-11-23T20:26:55.985320Z","iopub.status.idle":"2021-11-23T20:26:56.012080Z","shell.execute_reply.started":"2021-11-23T20:26:55.985290Z","shell.execute_reply":"2021-11-23T20:26:56.011508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss() # same as categorical_crossentropy loss used in Keras models which runs on Tensorflow\noptimizer = torch.optim.Adam(model.parameters(), lr = wandb.config['lr']) # fine tuned the lr","metadata":{"_uuid":"8901e464-4dca-4a27-b446-5019ec90f1f0","_cell_guid":"ed88c0fb-b3b8-456a-817f-7479d95399b7","collapsed":false,"id":"gaYkrdaGO0_A","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:28:27.216688Z","iopub.execute_input":"2021-11-23T20:28:27.217019Z","iopub.status.idle":"2021-11-23T20:28:27.223660Z","shell.execute_reply.started":"2021-11-23T20:28:27.216989Z","shell.execute_reply":"2021-11-23T20:28:27.222951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%wandb\n\nepochs = wandb.config['epochs']\nrunning_loss_history = []\nrunning_corrects_history = []\nval_running_loss_history = []\nval_running_corrects_history = []\n\nrun.watch(model)\nwandb_table = []\n\n\nfor e in range(epochs): # training our model, put input according to every batch.\n  \n    running_loss = 0.0\n    running_corrects = 0.0\n    val_running_loss = 0.0\n    val_running_corrects = 0.0\n\n    for inputs, labels in training_loader:\n        inputs = inputs.to(device) # input to device as our model is running in mentioned device.\n        labels = labels.to(device)\n        outputs = model(inputs) # every batch of 100 images are put as an input.\n        loss = criterion(outputs, labels) # Calc loss after each batch i/p by comparing it to actual labels. \n\n        optimizer.zero_grad() #setting the initial gradient to 0\n        loss.backward() # backpropagating the loss\n        optimizer.step() # updating the weights and bias values for every single step.\n\n        _, preds = torch.max(outputs, 1) # taking the highest value of prediction.\n        running_loss += loss.item()\n        running_corrects += torch.sum(preds == labels.data) # calculating te accuracy by taking the sum of all the correct predictions in a batch.\n\n    with torch.no_grad(): # we do not need gradient for validation.\n        for val_inputs, val_labels in validation_loader:\n            val_inputs = val_inputs.to(device)\n            val_labels = val_labels.to(device)\n            val_outputs = model(val_inputs)\n            val_loss = criterion(val_outputs, val_labels)\n            run.log({\n                'maximum_value': val_inputs.max().cpu().detach().numpy(),\n                'minimal_value_eval': val_inputs.min().cpu().detach().numpy(),\n                'mean_value_eval': val_inputs.mean().cpu().detach().numpy(),\n                'std_eval': val_inputs.std().cpu().detach().numpy(),\n            })\n            if random.random() < 0.01:\n                idx =  random.randint(0,len(val_inputs)-1)\n                wandb_table.append([wandb.Image(torch.permute(val_inputs[idx], (1, 2, 0)).cpu().detach().numpy()),val_labels[idx].cpu().detach().numpy() ,torch.argmax(val_outputs[idx]).cpu().detach().numpy() ,e])\n\n                    \n                \n\n        _, val_preds = torch.max(val_outputs, 1)\n        val_running_loss += val_loss.item()\n        val_running_corrects += torch.sum(val_preds == val_labels.data)\n\n        epoch_loss = running_loss/len(training_loader) # loss per epoch\n        epoch_acc = running_corrects.float()/ len(training_loader) # accuracy per epoch\n        running_loss_history.append(epoch_loss) # appending for displaying \n        running_corrects_history.append(epoch_acc)\n\n        val_epoch_loss = val_running_loss/len(validation_loader)\n        val_epoch_acc = val_running_corrects.float()/ len(validation_loader)\n        val_running_loss_history.append(val_epoch_loss)\n        val_running_corrects_history.append(val_epoch_acc)\n        \n        \n        run.log({\"train_loss\": epoch_loss, \"train_acc\":epoch_acc.item(), \"val_loss\": val_epoch_loss, 'val_acc': val_epoch_acc.item()})\n        \n        print('epoch :', (e+1))\n        print('training loss: {:.4f}, acc {:.4f} '.format(epoch_loss, epoch_acc.item()))\n        print('validation loss: {:.4f}, validation acc {:.4f} '.format(val_epoch_loss, val_epoch_acc.item()))","metadata":{"_uuid":"22c2d1c3-6d27-407e-945c-9f0038263c3b","_cell_guid":"26fff501-373a-4f33-ac86-4659e7d23da6","collapsed":false,"id":"wqIzL_FiPKto","outputId":"bee0b331-d45c-4a98-9448-a64a762bbdb1","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T20:34:17.920336Z","iopub.execute_input":"2021-11-23T20:34:17.921158Z","iopub.status.idle":"2021-11-23T20:45:26.365822Z","shell.execute_reply.started":"2021-11-23T20:34:17.921111Z","shell.execute_reply":"2021-11-23T20:45:26.364761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb_table = [[x[0], classes[int(x[1])], classes[int(x[2])], x[3]] for x in wandb_table]\n\nwandb_table_2 = [x for x in wandb_table]\nt = [\"Image\",\"actual label\", \"predicted label\", 'epoch']\n\nrun.log({\"custom_data_table\": wandb.Table(data=wandb_table_2,\n                                columns = t)})","metadata":{"_uuid":"a4737ea8-1d0b-4f20-8782-5ce4f1258d05","_cell_guid":"3b5d9d0a-1f66-4c9e-88b4-7473e0c67cb1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T19:47:42.438852Z","iopub.execute_input":"2021-11-23T19:47:42.439113Z","iopub.status.idle":"2021-11-23T19:47:43.106232Z","shell.execute_reply.started":"2021-11-23T19:47:42.439076Z","shell.execute_reply":"2021-11-23T19:47:43.105364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{"_uuid":"f1c57633-5182-4b5e-9bf4-ad965c41a75c","_cell_guid":"ba076857-7114-433a-82a1-afa442188f1b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-11-23T19:47:43.108585Z","iopub.execute_input":"2021-11-23T19:47:43.108917Z","iopub.status.idle":"2021-11-23T19:47:49.339133Z","shell.execute_reply.started":"2021-11-23T19:47:43.108872Z","shell.execute_reply":"2021-11-23T19:47:49.33846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"7f2cda45-f9f3-42de-94a5-332c24763620","_cell_guid":"ae725849-b1b4-4395-9c31-76169a70a183","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}