{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n\nPlankton Classification Challenge
\n This notebook has been prepared on the Plankton Classification Challenge .
\n This not only holds the solution but also is a nice
\n starter notebook for image classification with PyTorch
","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-09-19T06:11:21.615652Z","iopub.status.busy":"2021-09-19T06:11:21.614908Z","iopub.status.idle":"2021-09-19T06:11:21.70779Z","shell.execute_reply":"2021-09-19T06:11:21.707143Z","shell.execute_reply.started":"2021-09-19T06:11:21.61555Z"}}},{"cell_type":"markdown","source":"At first we need to understand the steps that how we are going to solve the whole challenge by conquering every small step.","metadata":{}},{"cell_type":"markdown","source":"# Libraries :\n---","metadata":{}},{"cell_type":"code","source":"# Manipulation Libraries\n\nfrom termcolor import cprint\nimport os\nfrom glob import glob\nimport random\nfrom warnings import filterwarnings\nfilterwarnings('ignore')\n\n\n# supporting libraries\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nfrom PIL import Image\nimport random\n\n#importing pytorch and associated libraries\n\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms as transforms\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:52:54.442302Z","iopub.execute_input":"2021-09-20T09:52:54.446221Z","iopub.status.idle":"2021-09-20T09:52:59.949650Z","shell.execute_reply.started":"2021-09-20T09:52:54.446061Z","shell.execute_reply":"2021-09-20T09:52:59.948760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # Unzipping data :\n ---\n \n Now we are unzipping the train, test and the sample submission to check which type of data we should be feeding to the model and what should be the output format.","metadata":{}},{"cell_type":"code","source":"# unzipping train folder\n!unzip -q ../input/datasciencebowl/train.zip","metadata":{"execution":{"iopub.status.busy":"2021-09-20T06:14:44.485076Z","iopub.execute_input":"2021-09-20T06:14:44.485349Z","iopub.status.idle":"2021-09-20T06:14:47.364475Z","shell.execute_reply.started":"2021-09-20T06:14:44.485315Z","shell.execute_reply":"2021-09-20T06:14:47.36357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unzipping test folder\n!unzip -q ../input/datasciencebowl/test.zip","metadata":{"execution":{"iopub.status.busy":"2021-09-20T06:14:47.368969Z","iopub.execute_input":"2021-09-20T06:14:47.369242Z","iopub.status.idle":"2021-09-20T06:14:57.300527Z","shell.execute_reply.started":"2021-09-20T06:14:47.369207Z","shell.execute_reply":"2021-09-20T06:14:57.299586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#unzipping sample submission file\n!unzip ../input/datasciencebowl/sampleSubmission.csv.zip","metadata":{"execution":{"iopub.status.busy":"2021-09-20T06:14:57.302889Z","iopub.execute_input":"2021-09-20T06:14:57.3032Z","iopub.status.idle":"2021-09-20T06:14:58.653208Z","shell.execute_reply.started":"2021-09-20T06:14:57.303159Z","shell.execute_reply":"2021-09-20T06:14:58.652382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After checking the input format we can see that the images of the train folder are stored in the folders as per their class names and the test data has only one folder containing all the images.\n\nThe sample submission has the name of the images and the the probabilty of the classes. So, we need to calculate the the softmax prediction of the classes.\n\nThus the problem can be achieved by a simple classification using a softmax layer at the bottom of the prediction.","metadata":{}},{"cell_type":"markdown","source":"# Class Distribution :\n---\n\n We should be aware of the class distributions cause every single class it's significance in this scenario.","metadata":{}},{"cell_type":"code","source":"# Class distribution in PIE_CHART \n\nclass_names = []\nclass_count = []\n\nfor name in os.listdir('./train/'):\n class_names.append(name)\n class_count.append(len(os.listdir(f'./train/{name}')))\nplt.figure(figsize=(10, 10))\nplt.pie(class_count, labels = class_names)\nplt.title('Class Distribution (Train)')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T06:14:58.656357Z","iopub.execute_input":"2021-09-20T06:14:58.656581Z","iopub.status.idle":"2021-09-20T06:15:00.12222Z","shell.execute_reply.started":"2021-09-20T06:14:58.656554Z","shell.execute_reply":"2021-09-20T06:15:00.121573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Class distribution in BAR-PLOT\n\nplt.figure(figsize=(20, 8))\nsns.barplot(class_names, class_count, palette = 'Blues')\nplt.title('Class Distribution (Train)')\nplt.xticks(rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-20T06:15:00.123094Z","iopub.execute_input":"2021-09-20T06:15:00.123477Z","iopub.status.idle":"2021-09-20T06:15:02.636452Z","shell.execute_reply.started":"2021-09-20T06:15:00.12344Z","shell.execute_reply":"2021-09-20T06:15:02.635762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fixing the seeds\n\ntorch.manual_seed(42)\nnp.random.seed(42)\nrandom.seed(42)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T06:15:02.637774Z","iopub.execute_input":"2021-09-20T06:15:02.63805Z","iopub.status.idle":"2021-09-20T06:15:02.645693Z","shell.execute_reply.started":"2021-09-20T06:15:02.638016Z","shell.execute_reply":"2021-09-20T06:15:02.645002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Generation :\n---\n\nThe model we are going to train for this chalenge will be a custom Convoltuional Neural Network. Primarily we will be updating some conv layers then we will flatten the whole image and then we will feed that through fully connected networks to generate outputs.","metadata":{}},{"cell_type":"code","source":"# Neural Network blocks and models\n\nclass Conv(nn.Module):\n def __init__(self, in_channels, out_channels, kerel_size = 3, stride = 1, padding = 0):\n super(Conv, self).__init__()\n self.seq = nn.Sequential(\n nn.Conv2d(in_channels, out_channels, kerel_size, stride, padding),\n nn.ReLU(),\n nn.BatchNorm2d(out_channels),\n )\n def forward(self, x):\n return self.seq(x)\n\nclass PlankNet(nn.Module):\n def __init__(self, in_channels, num_classes, H = 128, W = 128):\n super(PlankNet, self).__init__()\n self.model = nn.Sequential(\n Conv(in_channels, 16, 4), #125\n nn.MaxPool2d(2), #62\n Conv(16, 32, 3), #60\n Conv(32, 64, 3), #58\n nn.Dropout(0.1),\n nn.MaxPool2d(2), # 29\n Conv(64, 128), # 27\n nn.Dropout(0.2),\n Conv(128, 64, 3), # 25\n Conv(64, 32, 3), # 23\n nn.Flatten(),\n nn.Linear(32*23*23 , 4096),\n nn.Linear(4096, num_classes),\n )\n def forward(self, x):\n return self.model(x)\nnum_classes = len(class_names)\nmodel = PlankNet(3, num_classes, 128, 128)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:53:08.896552Z","iopub.execute_input":"2021-09-20T09:53:08.896884Z","iopub.status.idle":"2021-09-20T09:53:09.446985Z","shell.execute_reply.started":"2021-09-20T09:53:08.896853Z","shell.execute_reply":"2021-09-20T09:53:09.446152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand_data = torch.rand(1, 3, 128, 128)\nprint(model(rand_data).shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-20T09:53:09.448345Z","iopub.execute_input":"2021-09-20T09:53:09.448582Z","iopub.status.idle":"2021-09-20T09:53:09.562880Z","shell.execute_reply.started":"2021-09-20T09:53:09.448551Z","shell.execute_reply":"2021-09-20T09:53:09.561952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cprint(model, \"blue\")","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:48.81516Z","iopub.execute_input":"2021-09-19T18:10:48.815423Z","iopub.status.idle":"2021-09-19T18:10:48.821257Z","shell.execute_reply.started":"2021-09-19T18:10:48.815395Z","shell.execute_reply":"2021-09-19T18:10:48.820465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model layers overview\n\nfor name, param in model.named_parameters():\n print(f\"{name} : {param.shape}\")","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:50.169579Z","iopub.execute_input":"2021-09-19T18:10:50.170181Z","iopub.status.idle":"2021-09-19T18:10:50.181994Z","shell.execute_reply.started":"2021-09-19T18:10:50.17014Z","shell.execute_reply":"2021-09-19T18:10:50.180943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Generation :\n---\n\nNow we will be creating the dataset which will be fed through the CNN-Model.\nThe training data will be splitted in training and validation for checking the model performance.\n\nFirstly in this step we will create the dataframe eith which we will acces each data elements and it's additional features.\nThen we will create the dataset class and then fit that though dataloader which will create batches of data to feed into the model.","metadata":{}},{"cell_type":"code","source":"#Generating csv file to gather data of images \n\ndef generate_csv(root,train = True, img_ext = 'jpg'):\n df = pd.DataFrame(columns = ['path', 'class'])\n if train:\n for index,label in enumerate(os.listdir(root)):\n links = glob(f\"{root}/{label}/*{img_ext}\")\n temp_df = pd.DataFrame({'path': links, 'class': np.ones(len(links), dtype='float32')*index})\n df = pd.concat([df, temp_df], axis = 0)\n else:\n links = glob(f\"{root}/*{img_ext}\")\n temp_df = pd.DataFrame({'path': links, 'class': np.ones(len(links), dtype = 'float32')})\n df = pd.concat([df, temp_df], axis = 0)\n \n return df\n ","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:51.819442Z","iopub.execute_input":"2021-09-19T18:10:51.82Z","iopub.status.idle":"2021-09-19T18:10:51.82721Z","shell.execute_reply.started":"2021-09-19T18:10:51.819945Z","shell.execute_reply":"2021-09-19T18:10:51.826226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = generate_csv('./train')\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:52.244445Z","iopub.execute_input":"2021-09-19T18:10:52.244729Z","iopub.status.idle":"2021-09-19T18:10:52.529325Z","shell.execute_reply.started":"2021-09-19T18:10:52.2447Z","shell.execute_reply":"2021-09-19T18:10:52.528546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = generate_csv('./test',train = False)\ntest_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:52.623957Z","iopub.execute_input":"2021-09-19T18:10:52.624215Z","iopub.status.idle":"2021-09-19T18:10:53.063656Z","shell.execute_reply.started":"2021-09-19T18:10:52.624189Z","shell.execute_reply":"2021-09-19T18:10:53.062734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see that the names are not sorted as shown n the sample submission , so we need to sort the label names of the test data.","metadata":{}},{"cell_type":"code","source":"# sorting the image by their names\n\ntest_csv.sort_values('path', inplace = True)\ntest_csv.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:53.489278Z","iopub.execute_input":"2021-09-19T18:10:53.490094Z","iopub.status.idle":"2021-09-19T18:10:53.698107Z","shell.execute_reply.started":"2021-09-19T18:10:53.490047Z","shell.execute_reply":"2021-09-19T18:10:53.697274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Genrating dataset classes\n\ndef load_image(path , H, W):\n img = cv2.imread(path)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n img = cv2.resize(img, (H,W))\n return img\n\ndef transform():\n return transforms.Compose([\n transforms.RandomRotation(90),\n transforms.ToTensor()\n ])\n\n\nclass PlanktonDataset(Dataset):\n def __init__(self, df, H = 128, W = 128, transform = None):\n super(PlanktonDataset, self).__init__()\n self.df = df\n self.H = H\n self.W = W\n self.transform = transform\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, index):\n path = self.df.iloc[index, 0]\n img = load_image(path, H = self.H, W = self.W)\n label = self.df.iloc[index, 1]\n img = Image.fromarray(img)\n if self.transform != None:\n img = self.transform()(img)\n else:\n img = transforms.ToTensor()(img)\n return (img, (label, path))\n \n# Creating the train and test datasets. \ntrain_ds = PlanktonDataset(train_csv, 128, 128, transform = transform)\ntest_ds = PlanktonDataset(test_csv, 128, 128)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:54.129511Z","iopub.execute_input":"2021-09-19T18:10:54.130506Z","iopub.status.idle":"2021-09-19T18:10:54.140951Z","shell.execute_reply.started":"2021-09-19T18:10:54.130449Z","shell.execute_reply":"2021-09-19T18:10:54.140225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data access format\n\nimage, label = test_ds.__getitem__(3)\nimage.shape, label[0], label[1]","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:55.244756Z","iopub.execute_input":"2021-09-19T18:10:55.245298Z","iopub.status.idle":"2021-09-19T18:10:55.296775Z","shell.execute_reply.started":"2021-09-19T18:10:55.245262Z","shell.execute_reply":"2021-09-19T18:10:55.295953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The train-validation split is taken 3:1 (75% - 25%)","metadata":{}},{"cell_type":"code","source":"# Train-Validation Split :\n\nspl_idx = int(train_ds.__len__() * 0.75)\nprint(f\"Splitting index : {spl_idx}\")\ntrain_ds, val_ds = torch.utils.data.random_split(train_ds,[spl_idx, train_ds.__len__() - spl_idx] )","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:10:59.624449Z","iopub.execute_input":"2021-09-19T18:10:59.62472Z","iopub.status.idle":"2021-09-19T18:10:59.631894Z","shell.execute_reply.started":"2021-09-19T18:10:59.624691Z","shell.execute_reply":"2021-09-19T18:10:59.631208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Size of train dataset : {train_ds.__len__()}\")\nprint(f\"Size of validation dataset : {val_ds.__len__()}\")\nprint(f\"Size of test dataset : {test_ds.__len__()}\")","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:11:00.63965Z","iopub.execute_input":"2021-09-19T18:11:00.640267Z","iopub.status.idle":"2021-09-19T18:11:00.644926Z","shell.execute_reply.started":"2021-09-19T18:11:00.640227Z","shell.execute_reply":"2021-09-19T18:11:00.644279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating data loaders specifying the batch size\n\nBATCH_SIZE = 64\ntrain_dl = DataLoader(train_ds, batch_size = BATCH_SIZE, shuffle = True)\nval_dl = DataLoader(val_ds, batch_size = BATCH_SIZE, shuffle = True)\ntest_dl = DataLoader(test_ds, batch_size = BATCH_SIZE, shuffle = False)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:11:01.124499Z","iopub.execute_input":"2021-09-19T18:11:01.125233Z","iopub.status.idle":"2021-09-19T18:11:01.13072Z","shell.execute_reply.started":"2021-09-19T18:11:01.125194Z","shell.execute_reply":"2021-09-19T18:11:01.129677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Compilation, Hyperparameter Tuning , Training :\n---\n\nIn this step we'll be feeding the data to the CNN-Model. Do, keep in mind this trainer has been made to only surpass GPU devices and will not work on the cpu devices.\n\n**Note :** If you want to run the trainer on cpu , just comment the `.cuda()` portions.","metadata":{}},{"cell_type":"code","source":"# Chossing training hyperparameters and also the optimizer and loss\n\nEPOCHS = 30\ncriterion = nn.CrossEntropyLoss()\noptim = torch.optim.Adam(params = model.parameters(), lr = 1e-4)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:11:01.804522Z","iopub.execute_input":"2021-09-19T18:11:01.804792Z","iopub.status.idle":"2021-09-19T18:11:01.810145Z","shell.execute_reply.started":"2021-09-19T18:11:01.804763Z","shell.execute_reply":"2021-09-19T18:11:01.80927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loss = []\nval_loss = []","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:11:02.204323Z","iopub.execute_input":"2021-09-19T18:11:02.205212Z","iopub.status.idle":"2021-09-19T18:11:02.209298Z","shell.execute_reply.started":"2021-09-19T18:11:02.205166Z","shell.execute_reply":"2021-09-19T18:11:02.208584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model Training...\n\nmodel = model.cuda() # Putting the model inside GPU\nbest_loss = np.inf\nfor epoch in range(EPOCHS):\n print(f\"Epoch {epoch + 1} : \\n\")\n TR_LOSS = 0.0\n VAL_LOSS = 0.0\n model.train()\n \n # Train Data Forward & Backward Pass\n \n for index, (train_patch, (labels, _)) in enumerate(train_dl):\n optim.zero_grad()\n train_patch = train_patch.cuda()\n labels = labels.long().cuda()\n op = model(train_patch)\n tloss = criterion(op, labels)\n TR_LOSS += tloss.item()\n train_loss.append(tloss.item())\n tloss.backward()\n optim.step()\n \n if index % 100 == 99:\n print(f\" Step {index + 1} Loss : {'%.4f'%(tloss.item())}\") \n model.eval()\n \n # Validation Checking ( Only Forward Pass )\n \n with torch.no_grad():\n for index, (val_patch, (labels, _)) in enumerate(val_dl):\n val_patch = val_patch.cuda()\n labels = labels.long().cuda()\n op = model(val_patch)\n vloss = criterion(op, labels)\n VAL_LOSS += vloss.item()\n val_loss.append(vloss.item())\n print(f\"\\n Training Loss : {'%.4f'%(TR_LOSS)} || Validation Loss : {'%.4f'%(VAL_LOSS)}\\n\")\n \n if VAL_LOSS < best_loss : # Model Updationg\n cprint(\"Model Updation : Success!\\n\", 'green')\n torch.save(model, 'best_model.pth')\n best_loss = VAL_LOSS\n else:\n cprint(\"Model Updation : Failed!\\n\", 'red')\ncprint('Training completed...', 'green')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T18:11:03.274778Z","iopub.execute_input":"2021-09-19T18:11:03.275408Z","iopub.status.idle":"2021-09-19T18:21:58.835395Z","shell.execute_reply.started":"2021-09-19T18:11:03.275368Z","shell.execute_reply":"2021-09-19T18:21:58.83406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation :\n---\n\nNow, we should check if the model is overfitting or not. But in this scenario we can see that the loss function has too many local minima but the model hparams are quite successful to cross those and reach a satisfactory point.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,8))\nplt.plot(train_loss)\nplt.title('Train Loss', size = 20)\nplt.xlabel('STEPS')\nplt.ylabel('LOSS')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T08:53:17.507822Z","iopub.execute_input":"2021-09-19T08:53:17.508115Z","iopub.status.idle":"2021-09-19T08:53:17.76372Z","shell.execute_reply.started":"2021-09-19T08:53:17.50808Z","shell.execute_reply":"2021-09-19T08:53:17.763016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,8))\nplt.plot(val_loss)\nplt.title('Validation Loss', size = 20)\nplt.xlabel('STEPS')\nplt.ylabel('LOSS')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T08:53:17.76503Z","iopub.execute_input":"2021-09-19T08:53:17.76544Z","iopub.status.idle":"2021-09-19T08:53:17.990466Z","shell.execute_reply.started":"2021-09-19T08:53:17.765404Z","shell.execute_reply":"2021-09-19T08:53:17.989771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction :\n---\n\n In this step we will load the best fitted model and then cretae the submission dataframe and then poplate that data on a `.csv` file.","metadata":{}},{"cell_type":"code","source":"best_model = torch.load('./best_model.pth')\ncprint(best_model, 'blue')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T08:53:17.991716Z","iopub.execute_input":"2021-09-19T08:53:17.991963Z","iopub.status.idle":"2021-09-19T08:53:18.374623Z","shell.execute_reply.started":"2021-09-19T08:53:17.99193Z","shell.execute_reply":"2021-09-19T08:53:18.373848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to filter only the names of the images\n\ndef preprocess_names(names_list):\n name_list = []\n for name in names_list:\n name_list.append(name.split('/')[-1])\n return name_list","metadata":{"execution":{"iopub.status.busy":"2021-09-19T08:53:18.37598Z","iopub.execute_input":"2021-09-19T08:53:18.376436Z","iopub.status.idle":"2021-09-19T08:53:18.381679Z","shell.execute_reply.started":"2021-09-19T08:53:18.376398Z","shell.execute_reply":"2021-09-19T08:53:18.380857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test data generating function\n\ndef create_submission_file(test_dataloader, model, class_names):\n df = pd.DataFrame()\n model.eval()\n sf_layer = nn.Softmax(dim = 1)\n with torch.no_grad():\n for index, (test_patch, attr) in enumerate(test_dataloader): # loading test data\n paths = preprocess_names(attr[1])\n test_patch = test_patch.cuda()\n op = model(test_patch)\n op = sf_layer(op)\n paths = np.array(list(paths)) #adding the filenames in a list\n op = op.cpu().detach().numpy()\n patch_df = pd.DataFrame(op, columns = class_names) #creating the patch dataframe\n patch_df.insert(0, 'image', paths)\n df = pd.concat([df, patch_df], axis = 0) # joining the patch dataframe with the main one.\n \n if index % 100 == 99:\n print(f\"{index + 1} Steps Completed...\\n\") \n print('Test Dataframe Generated...\\n')\n \n return df","metadata":{"execution":{"iopub.status.busy":"2021-09-19T08:53:18.382877Z","iopub.execute_input":"2021-09-19T08:53:18.383503Z","iopub.status.idle":"2021-09-19T08:53:18.395469Z","shell.execute_reply.started":"2021-09-19T08:53:18.383465Z","shell.execute_reply":"2021-09-19T08:53:18.394736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = best_model.cuda()\npred_df = create_submission_file(test_dl, best_model, class_names)\npred_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T08:53:18.396557Z","iopub.execute_input":"2021-09-19T08:53:18.397271Z","iopub.status.idle":"2021-09-19T08:55:30.654731Z","shell.execute_reply.started":"2021-09-19T08:53:18.397233Z","shell.execute_reply":"2021-09-19T08:55:30.65381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we need to do a last sanity chcek if the dataframe is okay or not.","metadata":{}},{"cell_type":"code","source":"# Checking if all the imae names are different\nassert pred_df['image'].nunique() == pred_df.shape[0], \" Submission format not correct!\"\ncprint('Submission correcty created !', 'green')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T08:55:30.656384Z","iopub.execute_input":"2021-09-19T08:55:30.656672Z","iopub.status.idle":"2021-09-19T08:55:30.703529Z","shell.execute_reply.started":"2021-09-19T08:55:30.656636Z","shell.execute_reply":"2021-09-19T08:55:30.702757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating `submission.csv` file","metadata":{}},{"cell_type":"code","source":"pred_df.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T08:55:30.704899Z","iopub.execute_input":"2021-09-19T08:55:30.705163Z","iopub.status.idle":"2021-09-19T08:55:52.283719Z","shell.execute_reply.started":"2021-09-19T08:55:30.705128Z","shell.execute_reply":"2021-09-19T08:55:52.282957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"HURRAH!
","metadata":{}},{"cell_type":"markdown","source":"Now, We've completed the whole project. \n## Do ***UPVOTE*** this notebook.\nFollow me on [GitHub](https://github.com/sagnik1511) and also in [Kaggle](https://kaggle.com/sagnik1511).","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}