{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## References \n[Article](https://arxiv.org/pdf/1708.02002.pdf).  \n  \n [configuration](https://github.com/yhenon/pytorch-retinanet) [yhenon](https://github.com/yhenon).\n","metadata":{}},{"cell_type":"code","source":"### Cloning Github Repository \n!git clone https://github.com/yhenon/pytorch-retinanet.git","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-03T03:21:49.99467Z","iopub.execute_input":"2021-07-03T03:21:49.994993Z","iopub.status.idle":"2021-07-03T03:21:51.37694Z","shell.execute_reply.started":"2021-07-03T03:21:49.994963Z","shell.execute_reply":"2021-07-03T03:21:51.376192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Copying RetinaNet Folder to root dir so we can import it easily\n!cp -r /kaggle/working/pytorch-retinanet/retinanet ./","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:21:51.379078Z","iopub.execute_input":"2021-07-03T03:21:51.379498Z","iopub.status.idle":"2021-07-03T03:21:52.08608Z","shell.execute_reply.started":"2021-07-03T03:21:51.379454Z","shell.execute_reply":"2021-07-03T03:21:52.084834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pycocotools","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-03T03:21:52.087541Z","iopub.execute_input":"2021-07-03T03:21:52.087881Z","iopub.status.idle":"2021-07-03T03:22:06.32765Z","shell.execute_reply.started":"2021-07-03T03:21:52.087841Z","shell.execute_reply":"2021-07-03T03:22:06.326804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport cv2\nimport time\nimport numpy as np\nimport pandas as pd\n\n\nimport torch\nimport torch.optim as optim\nimport torchvision.transforms as T\nfrom torchvision.utils import make_grid \nfrom torch.utils.data import DataLoader, Dataset\n\nfrom retinanet import model\nfrom retinanet.dataloader import collater, Resizer, Augmenter, Normalizer, UnNormalizer\n\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\nDIR = \"../input/singapore-maritime-dataset/Singapore_maritime_dataset/\"\nDIR_TRAIN = DIR + \"train\"\nDIR_TEST = DIR + \"test\"","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:06.329072Z","iopub.execute_input":"2021-07-03T03:22:06.329569Z","iopub.status.idle":"2021-07-03T03:22:08.812365Z","shell.execute_reply.started":"2021-07-03T03:22:06.329522Z","shell.execute_reply":"2021-07-03T03:22:08.811209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring Dataset","metadata":{}},{"cell_type":"code","source":"### Loading Dataset\ndf = pd.read_csv(DIR + \"out3/train_labels.csv\")\ndf.head(50\n       )\n","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:08.820418Z","iopub.execute_input":"2021-07-03T03:22:08.823091Z","iopub.status.idle":"2021-07-03T03:22:08.992245Z","shell.execute_reply.started":"2021-07-03T03:22:08.823046Z","shell.execute_reply":"2021-07-03T03:22:08.99106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converting bbox list from original df to some appropriate form","metadata":{}},{"cell_type":"markdown","source":"Null Values, Unique Images, etc.","metadata":{}},{"cell_type":"code","source":"### Null Values, Unique Images, etc.\n\nunq_values = df[\"filename\"].unique()\nprint(\"Total Records: \", len(df))\nprint(\"Unique Images: \",len(unq_values))\n\nnull_values = df.isnull().sum(axis = 0)\nprint(\"\\n> Null Values in each column <\")\nprint(null_values)\n\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-03T03:22:08.99778Z","iopub.execute_input":"2021-07-03T03:22:09.000509Z","iopub.status.idle":"2021-07-03T03:22:09.030289Z","shell.execute_reply.started":"2021-07-03T03:22:09.000457Z","shell.execute_reply":"2021-07-03T03:22:09.029233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Data Sources\n\nsources = df[\"class\"].unique()\nprint(\"Total Sources: \",len(sources))\nprint(\"\\n> Sources <\\n\",sources)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-07-03T03:22:09.035226Z","iopub.execute_input":"2021-07-03T03:22:09.037987Z","iopub.status.idle":"2021-07-03T03:22:09.050493Z","shell.execute_reply.started":"2021-07-03T03:22:09.037944Z","shell.execute_reply":"2021-07-03T03:22:09.049455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Visualizing Source Distribution\n\nplt.figure(figsize=(14,8))\nplt.title('Source Distribution', fontsize= 20)\nsns.countplot(x = \"class\", data = df)","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:09.055005Z","iopub.execute_input":"2021-07-03T03:22:09.055719Z","iopub.status.idle":"2021-07-03T03:22:09.376155Z","shell.execute_reply.started":"2021-07-03T03:22:09.055683Z","shell.execute_reply":"2021-07-03T03:22:09.375083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Splitting Train Dataset into train - val (80:20)\n\nimages = df['filename'].unique()\nvalid_imgs = images[-300:]\ntrain_imgs = images[:-300]\n\nvalid_df = df[df['filename'].isin(valid_imgs)]\ntrain_df = df[df['filename'].isin(train_imgs)]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:09.377687Z","iopub.execute_input":"2021-07-03T03:22:09.378049Z","iopub.status.idle":"2021-07-03T03:22:09.395818Z","shell.execute_reply.started":"2021-07-03T03:22:09.378011Z","shell.execute_reply":"2021-07-03T03:22:09.395155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Random Images with BBox","metadata":{}},{"cell_type":"code","source":"### Function to plot image\n\ndef plot_img(image_name):\n    \n    fig, ax = plt.subplots(1, 2, figsize = (30, 20))\n    ax = ax.flatten()\n    \n    records = df[df['filename'] == image_name]\n    img_path = os.path.join(DIR_TRAIN, image_name)\n    \n    image = cv2.imread(img_path, cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n    image /= 255.0\n    image2 = image\n    \n    ax[0].set_title('Original Image')\n    ax[0].imshow(image)\n    \n    for idx, row in records.iterrows():\n        box = row[['xmin', 'ymin', 'width', 'height']].values\n        xmin = box[0]\n        ymin = box[1]\n        width = box[2]\n        height = box[3]\n        \n        cv2.rectangle(image2, (int(xmin),int(ymin)), (int(xmin + width),int(ymin + height)), (255,0,0), 3)\n    \n    ax[1].set_title('Image with Bondary Box')\n    ax[1].imshow(image2)\n\n    plt.show()\n    ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-03T03:22:09.397465Z","iopub.execute_input":"2021-07-03T03:22:09.397862Z","iopub.status.idle":"2021-07-03T03:22:09.411566Z","shell.execute_reply.started":"2021-07-03T03:22:09.397823Z","shell.execute_reply":"2021-07-03T03:22:09.410025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Pass any image id as parameter\n\nplot_img(\"MVI_1474_VIS_frame0.jpg\")\nplot_img(\"MVI_1448_VIS_Haze_frame180.jpg\")\n#plot_img(\"MVI_1474_VIS_frame135.jpg\")\nplot_img(\"MVI_1482_VIS_frame100.jpg\")","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:09.413444Z","iopub.execute_input":"2021-07-03T03:22:09.413942Z","iopub.status.idle":"2021-07-03T03:22:12.865831Z","shell.execute_reply.started":"2021-07-03T03:22:09.413886Z","shell.execute_reply":"2021-07-03T03:22:12.864992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing Dataset for Training","metadata":{"trusted":true}},{"cell_type":"code","source":"### Creating targets for model using Dataset Class\n\nclass GWD(Dataset):\n\n    def __init__(self, dataframe, image_dir, mode = \"train\", transforms = None):\n        \n        super().__init__()\n        self.image_ids = dataframe['filename'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.mode = mode\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n\n        # Retriving image id and records from df\n        image_id = self.image_ids[index]\n        records = self.df[self.df['filename'] == image_id]\n\n        # Loading Image\n        image = cv2.imread(f'{self.image_dir}/{image_id}', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n\n        # If mode is set to train, then only we create targets\n        if self.mode == \"train\" or self.mode == \"valid\":\n\n            # Converting xmin, ymin, w, h to x1, y1, x2, y2\n            boxes = np.zeros((records.shape[0], 5))\n            boxes[:, 0:4] = records[['xmin', 'ymin', 'width', 'height']].values\n            boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n            boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n            boxes[:, 4] = 1 # This is for label, as we have only 1 class, it is always 1\n            \n            # Applying Transforms\n            sample = {'img': image, 'annot': boxes}\n                \n            if self.transforms:\n                sample = self.transforms(sample)\n\n            return sample\n        \n        elif self.mode == \"test\":\n            \n            # We just need to apply transoforms and return image\n            if self.transforms:\n                \n                sample = {'img' : image}\n                sample = self.transforms(sample)\n                \n            return sample\n        \n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:12.86759Z","iopub.execute_input":"2021-07-03T03:22:12.867929Z","iopub.status.idle":"2021-07-03T03:22:13.031745Z","shell.execute_reply.started":"2021-07-03T03:22:12.867895Z","shell.execute_reply":"2021-07-03T03:22:13.03068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Preparing Datasets and Dataloaders for Training \n\n# Dataset Object\ntrain_dataset = GWD(train_df, DIR_TRAIN, mode = \"train\", transforms = T.Compose([Augmenter(), Normalizer(), Resizer()]))\nvalid_dataset = GWD(valid_df, DIR_TRAIN, mode = \"valid\", transforms = T.Compose([Normalizer(), Resizer()]))\n\n# DataLoaders\ntrain_data_loader = DataLoader(\n    train_dataset,\n    batch_size = 8,\n    shuffle = True,\n    num_workers = 4,\n    collate_fn = collater\n)\n\nvalid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size = 8,\n    shuffle = True,\n    num_workers = 4,\n    collate_fn = collater\n)\n\n\ntest_data_loader = DataLoader(\n    valid_dataset,\n    batch_size = 1,\n    shuffle = True,\n    num_workers = 4,\n    collate_fn = collater\n)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:13.03343Z","iopub.execute_input":"2021-07-03T03:22:13.033808Z","iopub.status.idle":"2021-07-03T03:22:13.048942Z","shell.execute_reply.started":"2021-07-03T03:22:13.033769Z","shell.execute_reply":"2021-07-03T03:22:13.048176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Model - RetinaNet 🔨","metadata":{}},{"cell_type":"code","source":"### Utilize GPU if available\n\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:13.050567Z","iopub.execute_input":"2021-07-03T03:22:13.050989Z","iopub.status.idle":"2021-07-03T03:22:13.098792Z","shell.execute_reply.started":"2021-07-03T03:22:13.050942Z","shell.execute_reply":"2021-07-03T03:22:13.097974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### I am using Pre-trained Resnet50 as backbone\n\nretinanet = model.resnet50(num_classes = 11, pretrained = True)\n\n# Loading Pre-trained model - if you load pre-trained model, comment above line.\n#retinanet = torch.load(\"path_to_.pt_file\")","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:13.100986Z","iopub.execute_input":"2021-07-03T03:22:13.101378Z","iopub.status.idle":"2021-07-03T03:22:21.816295Z","shell.execute_reply.started":"2021-07-03T03:22:13.101336Z","shell.execute_reply":"2021-07-03T03:22:21.815523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Preparing model for training\n\n# Defininig Optimizer\noptimizer = torch.optim.Adam(retinanet.parameters(), lr = 0.0001)\n\n# Learning Rate Scheduler\n#lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size = 5, gamma=0.5)\n\nretinanet.to(device)\n\n#No of epochs\nepochs = 20\n","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:21.817523Z","iopub.execute_input":"2021-07-03T03:22:21.817784Z","iopub.status.idle":"2021-07-03T03:22:21.884886Z","shell.execute_reply.started":"2021-07-03T03:22:21.817757Z","shell.execute_reply":"2021-07-03T03:22:21.884079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### One Epoch - Train\n\ndef train_one_epoch(epoch_num, train_data_loader):\n    \n    print(\"Epoch - {} Started\".format(epoch_num))\n    st = time.time()\n    \n    retinanet.train()\n    \n    epoch_loss = []\n\n    for iter_num, data in enumerate(train_data_loader):\n                \n        # Reseting gradients after each iter\n        optimizer.zero_grad()\n            \n        # Forward\n        classification_loss, regression_loss = retinanet([data['img'].cuda().float(), data['annot'].cuda().float()])\n                \n        # Calculating Loss\n        classification_loss = classification_loss.mean()\n        regression_loss = regression_loss.mean()\n\n        loss = classification_loss + regression_loss\n\n        if bool(loss == 0):\n            continue\n                \n        # Calculating Gradients\n        loss.backward()\n\n        # Gradient Clipping\n        torch.nn.utils.clip_grad_norm_(retinanet.parameters(), 0.1)\n                \n        # Updating Weights\n        optimizer.step()\n\n        #Epoch Loss\n        epoch_loss.append(float(loss))\n\n            \n        print(\n            'Epoch: {} | Iteration: {} | Classification loss: {:1.5f} | Regression loss: {:1.5f} | Running loss: {:1.5f}'.format(\n                epoch_num, iter_num, float(classification_loss), float(regression_loss), np.mean(epoch_loss)))\n\n        del classification_loss\n        del regression_loss\n        \n    # Update the learning rate\n    #if lr_scheduler is not None:\n        #lr_scheduler.step()\n        \n    et = time.time()\n    print(\"\\n Total Time - {}\\n\".format(int(et - st)))\n    \n        ","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:21.886438Z","iopub.execute_input":"2021-07-03T03:22:21.886912Z","iopub.status.idle":"2021-07-03T03:22:21.898436Z","shell.execute_reply.started":"2021-07-03T03:22:21.886872Z","shell.execute_reply":"2021-07-03T03:22:21.897576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### One Epoch - Valid\n\ndef valid_one_epoch(epoch_num, valid_data_loader):\n    \n    print(\"Epoch - {} Started\".format(epoch_num))\n    st = time.time()\n    \n    epoch_loss = []\n\n    for iter_num, data in enumerate(valid_data_loader):\n                \n        with torch.no_grad():\n            \n            # Forward\n            classification_loss, regression_loss = retinanet([data['img'].cuda().float(), data['annot'].cuda().float()])\n\n            # Calculating Loss\n            classification_loss = classification_loss.mean()\n            regression_loss = regression_loss.mean()\n            loss = classification_loss + regression_loss\n\n            #Epoch Loss\n            epoch_loss.append(float(loss))\n\n            print(\n                'Epoch: {} | Iteration: {} | Classification loss: {:1.5f} | Regression loss: {:1.5f} | Running loss: {:1.5f}'.format(\n                    epoch_num, iter_num, float(classification_loss), float(regression_loss), np.mean(epoch_loss)))\n\n            del classification_loss\n            del regression_loss\n        \n    et = time.time()\n    print(\"\\n Total Time - {}\\n\".format(int(et - st)))\n    \n    # Save Model after each epoch\n    torch.save(retinanet, \"retinanet_gwd.pt\")\n    \n        ","metadata":{"execution":{"iopub.status.busy":"2021-07-03T03:22:21.900321Z","iopub.execute_input":"2021-07-03T03:22:21.900729Z","iopub.status.idle":"2021-07-03T03:22:21.913948Z","shell.execute_reply.started":"2021-07-03T03:22:21.900663Z","shell.execute_reply":"2021-07-03T03:22:21.913033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Training Loop\nfor epoch in range(epochs):\n    \n    # Call train function\n    train_one_epoch(epoch, train_data_loader)\n    \n    # Call valid function\n    #valid_one_epoch(epoch, valid_data_loader)\n","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-03T03:22:21.915857Z","iopub.execute_input":"2021-07-03T03:22:21.916254Z","iopub.status.idle":"2021-07-03T05:13:55.128644Z","shell.execute_reply.started":"2021-07-03T03:22:21.916216Z","shell.execute_reply":"2021-07-03T05:13:55.126946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Sample Results\nretinanet.eval()\nunnormalize = UnNormalizer()\n\nfor iter_num, data in enumerate(test_data_loader):\n    \n    # Getting Predictions\n    scores, classification, transformed_anchors = retinanet(data['img'].cuda().float())\n    \n    idxs = np.where(scores.cpu()>0.5)\n    img = np.array(255 * unnormalize(data['img'][0, :, :, :])).copy()\n    \n    img[img<0] = 0\n    img[img>255] = 255\n\n    img = np.transpose(img, (1, 2, 0))\n\n    img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_BGR2RGB)\n    \n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n    \n    for j in range(idxs[0].shape[0]):\n        bbox = transformed_anchors[idxs[0][j], :]\n        x1 = int(bbox[0])\n        y1 = int(bbox[1])\n        x2 = int(bbox[2])\n        y2 = int(bbox[3])\n\n        cv2.rectangle(img, (x1, y1), (x2, y2), color = (0, 0, 255), thickness = 2)\n        \n    ax.imshow(img)\n    \n    break\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-03T05:13:55.130066Z","iopub.status.idle":"2021-07-03T05:13:55.130878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}