{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":8089,"databundleVersionId":44321,"sourceType":"competition"},{"sourceId":24242,"sourceType":"datasetVersion","datasetId":18570},{"sourceId":24334,"sourceType":"datasetVersion","datasetId":18641},{"sourceId":193614080,"sourceType":"kernelVersion"}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Intro\nThis piece of work is inspired by a [youtube video lecture](https://www.youtube.com/watch?v=vaL1I2BD_xY) by Martin Gorner, a Google developer who put forth this idea of combining YOLO grid with SqueezeNet (instead of DarkNet - original stack for [YOLO](https://arxiv.org/pdf/1506.02640.pdf)).","metadata":{"_cell_guid":"e1f0f9ef-b2c2-4fe0-8098-24bd8b6cbbf9","_uuid":"6d83a1815b19019b63290af1a21834be59701070","execution":{"iopub.status.busy":"2024-08-05T09:46:16.543259Z","iopub.execute_input":"2024-08-05T09:46:16.543570Z","iopub.status.idle":"2024-08-05T09:46:16.548317Z","shell.execute_reply.started":"2024-08-05T09:46:16.543539Z","shell.execute_reply":"2024-08-05T09:46:16.547359Z"}}},{"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport tensorflow as tf\nimport random\nimport math\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport matplotlib.ticker as ticker\nimport copy\nfrom itertools import chain\nfrom skimage.io import imread, imshow, imread_collection, concatenate_images\nfrom skimage.transform import resize\nfrom skimage.morphology import label\nimport glob\nfrom zipfile import ZipFile \n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport torchvision\n# from torcheval.metrics.functional import binary_auroc\nimport torch.nn.functional as F\n\nimport time\nimport gc\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, StratifiedGroupKFold \n\n# For Image Models\nimport timm\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\n# For descriptive error messages\n# os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"_cell_guid":"04e6a10f-9963-48e1-9f87-def2cf0c7616","_uuid":"ace99be745c98fc8919e877a991adf5bde701ac3","execution":{"iopub.status.busy":"2024-08-26T18:03:21.347851Z","iopub.execute_input":"2024-08-26T18:03:21.348693Z","iopub.status.idle":"2024-08-26T18:03:29.130647Z","shell.execute_reply.started":"2024-08-26T18:03:21.348657Z","shell.execute_reply":"2024-08-26T18:03:29.129864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    \"seed\": 42,\n    \"image_size\": 384,\n    \"grid_dim\": 12,\n    \"mask_count\": 1,\n    \"max_bbox_count\": 2,\n    \"epochs\": 25,\n    \"train_batch_size\": 16,\n    \"valid_batch_size\": 16,\n    \"learning_rate\": 1e-4,\n    \"min_lr\": 1e-6,\n    \"weight_decay\": 1e-6,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n}","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:03:29.132783Z","iopub.execute_input":"2024-08-26T18:03:29.133260Z","iopub.status.idle":"2024-08-26T18:03:29.161202Z","shell.execute_reply.started":"2024-08-26T18:03:29.133224Z","shell.execute_reply":"2024-08-26T18:03:29.160286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:03:29.162535Z","iopub.execute_input":"2024-08-26T18:03:29.162891Z","iopub.status.idle":"2024-08-26T18:03:29.174951Z","shell.execute_reply.started":"2024-08-26T18:03:29.162855Z","shell.execute_reply":"2024-08-26T18:03:29.174034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = \"/kaggle/input/data-science-bowl-2018\"\nWORKING_DIR = \"/kaggle/working\"\nTRAIN_PATH = f'{WORKING_DIR}/train'\nTEST_PATH = f'{WORKING_DIR}/test'","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:03:29.176439Z","iopub.execute_input":"2024-08-26T18:03:29.176775Z","iopub.status.idle":"2024-08-26T18:03:29.189047Z","shell.execute_reply.started":"2024-08-26T18:03:29.176745Z","shell.execute_reply":"2024-08-26T18:03:29.188163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# specifying the zip file name \ntrain_zip_file = f\"{ROOT_DIR}/stage1_train.zip\"\ntest_zip_file = f\"{ROOT_DIR}/stage1_test.zip\"\n\n# opening the zip file in READ mode \nwith ZipFile(train_zip_file, 'r') as zip: \n    # printing all the contents of the zip file \n#     zip.printdir() \n  \n    # extracting all the files \n    print('Extracting all the files for training...') \n    zip.extractall(path=TRAIN_PATH) \n    print('Done!') \n    \n# opening the zip file in READ mode \nwith ZipFile(test_zip_file, 'r') as zip: \n    # printing all the contents of the zip file \n#     zip.printdir() \n  \n    # extracting all the files \n    print('Extracting all the files for testing...') \n    zip.extractall(path=TEST_PATH) \n    print('Done!') ","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:03:29.191900Z","iopub.execute_input":"2024-08-26T18:03:29.192455Z","iopub.status.idle":"2024-08-26T18:03:34.467031Z","shell.execute_reply.started":"2024-08-26T18:03:29.192430Z","shell.execute_reply":"2024-08-26T18:03:34.466071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_width = CONFIG[\"image_size\"]\nimg_height = CONFIG[\"image_size\"]\n\ngrid_dim = CONFIG['grid_dim']\ngrid_pix = img_width // grid_dim\n\nbatch_size = CONFIG['train_batch_size']\n\nmask_count = CONFIG['mask_count']\n#Set maximum bounding boxes allowed per grid cell\nmax_bbox_count = CONFIG['max_bbox_count']","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:03:34.468388Z","iopub.execute_input":"2024-08-26T18:03:34.469130Z","iopub.status.idle":"2024-08-26T18:03:34.474238Z","shell.execute_reply.started":"2024-08-26T18:03:34.469093Z","shell.execute_reply":"2024-08-26T18:03:34.473274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(img_width, grid_dim, grid_pix, batch_size, mask_count, max_bbox_count)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:03:34.475252Z","iopub.execute_input":"2024-08-26T18:03:34.475520Z","iopub.status.idle":"2024-08-26T18:03:34.490208Z","shell.execute_reply.started":"2024-08-26T18:03:34.475498Z","shell.execute_reply":"2024-08-26T18:03:34.489324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = next(os.walk(TRAIN_PATH))[1]\ntest_ids = next(os.walk(TEST_PATH))[1]\n\ntrain_images = np.zeros((len(train_ids), img_height, img_width, 3), dtype=np.uint8)\ntest_images = np.zeros((len(test_ids), img_height, img_width, 3), dtype=np.uint8)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:03:34.491273Z","iopub.execute_input":"2024-08-26T18:03:34.491541Z","iopub.status.idle":"2024-08-26T18:03:34.501753Z","shell.execute_reply.started":"2024-08-26T18:03:34.491519Z","shell.execute_reply":"2024-08-26T18:03:34.501057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resize train images.\nprint('resize train images... ')\nsys.stdout.flush()\n\nfor n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):\n    path = TRAIN_PATH + \"/\" + id_\n    img = imread(path + '/images/' + id_ + '.png')[:, :, :3]\n    img = resize(img, (img_height, img_width), mode='constant', preserve_range=True)\n\n    train_images[n] = img\n\nprint('Done!')","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:03:34.502691Z","iopub.execute_input":"2024-08-26T18:03:34.502915Z","iopub.status.idle":"2024-08-26T18:04:04.587334Z","shell.execute_reply.started":"2024-08-26T18:03:34.502895Z","shell.execute_reply":"2024-08-26T18:04:04.586397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_ids), len(train_images)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.588574Z","iopub.execute_input":"2024-08-26T18:04:04.588877Z","iopub.status.idle":"2024-08-26T18:04:04.595906Z","shell.execute_reply.started":"2024-08-26T18:04:04.588851Z","shell.execute_reply":"2024-08-26T18:04:04.594919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def store_bounding_boxes(img, train_id, mask_id, rotby_90):\n    ret, thresh = cv2.threshold(img, 127, 255, 0)\n    contours, _ = cv2.findContours(thresh.astype(np.uint8), 1, 2)\n    cnt = contours[0]\n#     print(cv2.boundingRect(cnt))\n    x, y, w, h = cv2.boundingRect(cnt)    \n    \n    x = x * (img_width/img.shape[1])\n    w = w * (img_width/img.shape[1])\n    y = y * (img_width/img.shape[0])\n    h = h * (img_width/img.shape[0])\n    \n    if(x > img_width-1):\n        x = img_width-1\n    if(y > img_height-1):\n        y = img_height-1\n    if(x+w > img_width-1):\n        w = img_width-1 - x\n    if(y+h > img_height-1):\n        h = img_height-1 - y\n        \n    bbdict = { \"train_id\": train_id, \"mask_id\": mask_id, \"rotby_90\": rotby_90, \"x\": x, \"y\": y, \"w\": w, \"h\": h}\n    return bbdict","metadata":{"_cell_guid":"f323fcc1-f5ad-49da-ae66-9114f02d318b","_uuid":"05c2163cc42f5cfbc0064f4bcf2ee542ee3d10b6","execution":{"iopub.status.busy":"2024-08-26T18:04:04.597303Z","iopub.execute_input":"2024-08-26T18:04:04.597661Z","iopub.status.idle":"2024-08-26T18:04:04.608578Z","shell.execute_reply.started":"2024-08-26T18:04:04.597631Z","shell.execute_reply":"2024-08-26T18:04:04.607739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_bboxes_csv = '/kaggle/input/d/vijaybj/data-science-bowl-2018/bboxes.csv'\nif not os.path.isfile(path_bboxes_csv):\n    bboxes = pd.DataFrame(columns=[\"train_id\", \"mask_id\", \"rotby_90\", \"x\", \"y\", \"w\", \"h\"])\n    row_count = 1\n    for n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):\n        path = TRAIN_PATH + \"/\" + id_\n        for mask_id, mask_file in enumerate(next(os.walk(path + '/masks/'))[2]):\n            mask_ = imread(path + '/masks/' + mask_file)\n    #         print(mask_[9])\n            for r in range(4):\n                bboxes.loc[row_count] = store_bounding_boxes(np.rot90(mask_, r), id_, mask_id, r)\n                row_count += 1\n    bboxes.to_csv('bboxes.csv', index=False)\nelse:\n    bboxes = pd.read_csv(path_bboxes_csv)","metadata":{"_cell_guid":"8bf9d677-d161-4965-98f1-1898235bfb3e","_uuid":"e7e51d42966cbfb60988aaae2c97d27ed036e16e","execution":{"iopub.status.busy":"2024-08-26T18:04:04.609568Z","iopub.execute_input":"2024-08-26T18:04:04.609825Z","iopub.status.idle":"2024-08-26T18:04:04.783243Z","shell.execute_reply.started":"2024-08-26T18:04:04.609793Z","shell.execute_reply":"2024-08-26T18:04:04.782410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_ids_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.784454Z","iopub.execute_input":"2024-08-26T18:04:04.784806Z","iopub.status.idle":"2024-08-26T18:04:04.789193Z","shell.execute_reply.started":"2024-08-26T18:04:04.784773Z","shell.execute_reply":"2024-08-26T18:04:04.788219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bboxes['grid_row'] = bboxes['y'] // grid_pix\nbboxes['grid_column'] = bboxes['x'] // grid_pix\n\nbboxes['grid_center_x'] = bboxes['grid_column'] * grid_pix + grid_pix/2\nbboxes['grid_center_y'] = bboxes['grid_row'] * grid_pix + grid_pix/2\n\nbboxes['box_center_x'] = bboxes.x + bboxes['w']/2\nbboxes['box_center_y'] = bboxes.y + bboxes['h']/2\n\nbboxes['new_x'] = (bboxes.box_center_x - bboxes.grid_center_x)/(img_width)\nbboxes['new_y'] = (bboxes.box_center_y - bboxes.grid_center_y)/(img_height)\n\nbboxes['new_w'] = np.sqrt(bboxes.w/(img_width))\nbboxes['new_h'] = np.sqrt(bboxes.h/(img_width))\n\nbboxes['confidence'] = 1\n\nbboxes['box_area'] = bboxes.new_w*bboxes.new_h","metadata":{"scrolled":true,"_cell_guid":"57f6e469-ff8f-4640-9c49-066230d172b4","_uuid":"34f67397be5647147d935e46c1d4c2676a2b329f","execution":{"iopub.status.busy":"2024-08-26T18:04:04.794448Z","iopub.execute_input":"2024-08-26T18:04:04.794713Z","iopub.status.idle":"2024-08-26T18:04:04.822028Z","shell.execute_reply.started":"2024-08-26T18:04:04.794690Z","shell.execute_reply":"2024-08-26T18:04:04.821321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bboxes.isnull().values.any()","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.822942Z","iopub.execute_input":"2024-08-26T18:04:04.823248Z","iopub.status.idle":"2024-08-26T18:04:04.826855Z","shell.execute_reply.started":"2024-08-26T18:04:04.823224Z","shell.execute_reply":"2024-08-26T18:04:04.826026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_grid_info(tr_id, rotby_90):\n    df = bboxes.loc[(bboxes.train_id == tr_id) & (bboxes.rotby_90 == rotby_90), 'grid_row':'box_area']\n    df.drop(['grid_center_x', 'grid_center_y','box_center_x', 'box_center_y',], axis = 1, inplace=True)\n    df = df.sort_values(['grid_column', 'grid_row', 'box_area'], ascending=False)\n    #print(len(df))\n    global mask_count\n    mask_count += len(df)\n    label_info = np.zeros(shape=(grid_dim, grid_dim, max_bbox_count, 5),  dtype=np.float32) + 0.000001\n    \n    for ind, row in df.iterrows():\n        i = int(row[0])\n        j = int(row[1])\n        for b in range(max_bbox_count):\n            if(label_info[i, j, b][4] != 1.0):\n                label_info[i, j, b] = np.array(row[2:7])\n                break\n    return label_info","metadata":{"_cell_guid":"bebafd3e-3c9e-4553-8951-12bc50c9275a","_uuid":"d98d67b7a65591259de624a4e684d8c66146e2d1","execution":{"iopub.status.busy":"2024-08-26T18:04:04.828100Z","iopub.execute_input":"2024-08-26T18:04:04.828502Z","iopub.status.idle":"2024-08-26T18:04:04.837628Z","shell.execute_reply.started":"2024-08-26T18:04:04.828469Z","shell.execute_reply":"2024-08-26T18:04:04.836878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_bboxes(img, bbs1):\n    print(img.shape)\n    f, axs = plt.subplots(1,2)\n    axs[0].imshow(img)\n    for i, b in enumerate(bbs1):\n        cv2.rectangle(img,(b[0], b[1]),(b[2], b[3]),(0,255,0),2)\n\n    axs[1].imshow(img)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.838604Z","iopub.execute_input":"2024-08-26T18:04:04.838840Z","iopub.status.idle":"2024-08-26T18:04:04.847967Z","shell.execute_reply.started":"2024-08-26T18:04:04.838819Z","shell.execute_reply":"2024-08-26T18:04:04.847210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_data(img, boxes1, thresh):\n    print(boxes1.shape)\n#     boxes1 = boxes1.cpu()\n    boxes1 = np.reshape(boxes1, newshape=[grid_dim, grid_dim, max_bbox_count, 5])\n    bbs1 = []\n\n    for i in range(grid_dim):\n        for j in range(grid_dim):\n            for b in range(max_bbox_count):\n                if(boxes1[i][j][b][4] > thresh):\n    #                 print('boxes1[i][j][b][4] == boxes2[i][j][b][4]:', boxes1[i][j][b][4])\n\n                    grid_center_x = ((j+0)*grid_pix + grid_pix/2)\n                    grid_center_y = ((i+0)*grid_pix + grid_pix/2)\n    #                 print(boxes1[i][j][b] == boxes2[i][j][b])\n\n                    new_box_center_x = boxes1[i][j][b][0] * img_width + grid_center_x\n                    new_box_center_y = boxes1[i][j][b][1] * img_height + grid_center_y\n\n                    new_w = np.square(boxes1[i][j][b][2]) * img_width\n                    new_h = np.square(boxes1[i][j][b][3]) * img_height\n\n                    x1 = new_box_center_x - new_w/2\n                    y1 = new_box_center_y - new_h/2\n\n                    x2 = new_box_center_x + new_w/2\n                    y2 = new_box_center_y + new_h/2\n\n                    bbs1.append((math.floor(x1), math.floor(y1), math.ceil(x2), math.ceil(y2)))\n                    \n    draw_bboxes(img, bbs1)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.849027Z","iopub.execute_input":"2024-08-26T18:04:04.849317Z","iopub.status.idle":"2024-08-26T18:04:04.860111Z","shell.execute_reply.started":"2024-08-26T18:04:04.849294Z","shell.execute_reply":"2024-08-26T18:04:04.859292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To build a custom dataset in PyTorch without downloading images:\n# Initialize the class with the necessary arguments, such as the list of image paths, labels, and transformations.\n# Implement the __len__ method to return the number of samples in the dataset.\n# Implement the __getitem__ method to return a single sample (image and label) from the dataset.\n\nimport torch\nfrom torch.utils.data import Dataset\nfrom PIL import Image\nfrom torchvision import transforms\n\nclass TrainDataset(Dataset):\n    def __init__(self, train_images, train_ids, transform=None):\n        self.train_images = train_images\n        self.train_ids = train_ids\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.train_images)\n\n    def __getitem__(self, index):\n#         print('index', index)\n        rotation = random.randint(0, 3)\n        img = self.__get_image(index, rotation)\n        label = self.__get_label(index, rotation)\n#         print('index', index)\n#         print('dataset image and label shapes', img.shape, label.shape)\n        return {\n            'image': img,\n            'label': label\n        }\n    \n    def __get_image(self, index, rotation):\n        image = np.rot90(self.train_images[index], rotation)\n#         image = train_images[index]\n        image = torch.from_numpy(image.copy())\n        return image\n\n    def __get_label(self, index, rotation):\n        grid_info = get_grid_info(train_ids[index], rotation)\n        label = torch.from_numpy(grid_info)\n        return label","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.861056Z","iopub.execute_input":"2024-08-26T18:04:04.861313Z","iopub.status.idle":"2024-08-26T18:04:04.874489Z","shell.execute_reply.started":"2024-08-26T18:04:04.861291Z","shell.execute_reply":"2024-08-26T18:04:04.873773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TrainDataset(train_images=train_images, train_ids=train_ids)\ntrain_dataloader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n                          num_workers=0, shuffle=True, pin_memory=True, drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.875326Z","iopub.execute_input":"2024-08-26T18:04:04.875610Z","iopub.status.idle":"2024-08-26T18:04:04.888544Z","shell.execute_reply.started":"2024-08-26T18:04:04.875587Z","shell.execute_reply":"2024-08-26T18:04:04.887652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data = next(iter(train_dataloader))\n# image = data['image'][0].numpy().copy()\n# label = data['label'][0].numpy().copy()\n\n# draw_data(image, label, thresh=0.1)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.889717Z","iopub.execute_input":"2024-08-26T18:04:04.890515Z","iopub.status.idle":"2024-08-26T18:04:04.898107Z","shell.execute_reply.started":"2024-08-26T18:04:04.890490Z","shell.execute_reply":"2024-08-26T18:04:04.897256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(train_dataloader)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.899103Z","iopub.execute_input":"2024-08-26T18:04:04.899341Z","iopub.status.idle":"2024-08-26T18:04:04.908751Z","shell.execute_reply.started":"2024-08-26T18:04:04.899321Z","shell.execute_reply":"2024-08-26T18:04:04.907931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_yolo_loss(y_pred, labels):\n    lambda_coords = 5.0\n    lambda_noobj = 0.5\n\n    # Compute squared difference between predictions and labels\n    yolo_loss = torch.sum(torch.square(labels - y_pred), dim=0)\n    yolo_loss = torch.sum(yolo_loss, dim=0)\n    yolo_loss = torch.sum(yolo_loss, dim=0)\n    yolo_loss = torch.sum(yolo_loss, dim=0)\n#     print('yolo loss before stacking', yolo_loss)\n    \n    # Separate the loss components and apply the appropriate lambda weights\n    yolo_loss = torch.stack([\n        lambda_coords * yolo_loss[0],\n        lambda_coords * yolo_loss[1],\n        yolo_loss[2],\n        yolo_loss[3],\n        lambda_noobj * yolo_loss[4]\n    ])\n#     print('yolo loss before final loss', yolo_loss)\n    \n    # Sum the losses to get the total loss\n    yolo_loss = torch.sum(yolo_loss)\n#     print('final yolo loss', yolo_loss)\n\n    return yolo_loss","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.909744Z","iopub.execute_input":"2024-08-26T18:04:04.910081Z","iopub.status.idle":"2024-08-26T18:04:04.920272Z","shell.execute_reply.started":"2024-08-26T18:04:04.910053Z","shell.execute_reply":"2024-08-26T18:04:04.919474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class YoloLogitsLayer(nn.Module):\n    def __init__(self):\n        super(YoloLogitsLayer, self).__init__()\n        self.avg_pool_3d = nn.AvgPool3d((1, 1, 16), stride=(1, 1, 16), padding=0)\n        self.grid_dim = 12\n        self.max_bb_cnt = 2\n\n    def forward(self, inputs):\n#         print(inputs.shape)\n        net = inputs.reshape(-1, 1, self.grid_dim, self.grid_dim, self.max_bb_cnt * 5 * 16)\n#         print(f\"input shape before avg 3d pool {net.shape}\")\n        net = self.avg_pool_3d(net)\n#         print(f\"input shape after avg 3d pool {net.shape}\")\n        net = net.permute(0, 2, 3, 4, 1)\n        net = net.reshape(-1, self.grid_dim * self.grid_dim * self.max_bb_cnt, 5)\n#         print(f\"input shape after second reshape {net.shape}\")\n\n        net = net.permute(1, 2, 0)\n#         print('before stacking:', net.shape)\n#         logits_tensor = stack_logits(net)\n#         logits_tensor = torch.stack([stack_logits(x) for x in net])\n#         print(f\"input shape before stacking {net.shape}\")\n\n        logits_tensor = torch.stack([\n            torch.tanh(net[:, 0]),\n            torch.tanh(net[:, 1]),\n            torch.sqrt(torch.sigmoid(net[:, 2])),\n            torch.sqrt(torch.sigmoid(net[:, 3])),\n            torch.sigmoid(net[:, 4])\n        ], dim=1)\n#         print('after stacking:', net.shape)\n#         print(f\"input shape before last permute {logits_tensor.shape}\")\n\n        logits_tensor = logits_tensor.permute(2, 0, 1)\n#         print(f\"input shape after last permute {logits_tensor.shape}\")\n\n        logits_tensor = logits_tensor.reshape(-1, self.grid_dim, self.grid_dim, self.max_bb_cnt, 5)\n#         print(f\"input shape after last reshape {logits_tensor.shape}\")\n        return logits_tensor","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:04:04.921240Z","iopub.execute_input":"2024-08-26T18:04:04.921720Z","iopub.status.idle":"2024-08-26T18:04:04.935280Z","shell.execute_reply.started":"2024-08-26T18:04:04.921696Z","shell.execute_reply":"2024-08-26T18:04:04.934400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FireModule(nn.Module):\n    def __init__(self, in_channels, squeeze_channels, expand_channels):\n        super(FireModule, self).__init__()\n        self.squeeze = nn.Conv2d(in_channels, squeeze_channels, kernel_size=1, padding='same')\n        self.expand1 = nn.Conv2d(squeeze_channels, expand_channels, kernel_size=1, padding='same')\n        self.expand3 = nn.Conv2d(squeeze_channels, expand_channels, kernel_size=3, padding='same')\n        self.leaky_relu = nn.LeakyReLU(negative_slope=0.1)\n#         self.batch_norm = v\n\n    def forward(self, x):\n#         print(f'Fire module forward: x.shape: {x.shape}')\n        squeeze = self.leaky_relu(self.squeeze(x))\n        expand1 = self.leaky_relu(self.expand1(squeeze))\n        expand3 = self.leaky_relu(self.expand3(squeeze))\n#         print('expand 1 and 3 shapes', expand1.shape, expand3.shape)\n        \n        return torch.cat([expand1, expand3], 1)\n\n    \nclass PermuteLayer(nn.Module):\n    def __init__(self, *dims):\n        super(PermuteLayer, self).__init__()\n        self.dims = dims\n\n    def forward(self, x):\n        return x.permute(*self.dims)\n\n    \nclass YoloModel(nn.Module):\n    def __init__(self):\n        super(YoloModel, self).__init__()\n        self.logits_layer = YoloLogitsLayer()\n\n        self.model = nn.Sequential(\n            nn.Conv2d(3, 32, kernel_size=1, stride=1, padding='same'),\n            nn.LeakyReLU(negative_slope=0.1),\n            \n            nn.MaxPool2d(kernel_size=2, stride=2, padding=0),\n            FireModule(32, 32, 64),\n            FireModule(128, 32, 64),\n            FireModule(128, 32, 64),\n\n            nn.MaxPool2d(kernel_size=2, stride=2, padding=0),\n            FireModule(128, 64, 96),\n            FireModule(192, 64, 96),\n            FireModule(192, 64, 96),\n\n            nn.MaxPool2d(kernel_size=2, stride=2, padding=0),\n            FireModule(192, 128, 160),\n            FireModule(320, 128, 160),\n            FireModule(320, 128, 128),\n\n            nn.MaxPool2d(kernel_size=2, stride=2, padding=0),\n            FireModule(256, 256, 512),\n            FireModule(1024, 256, 512),\n            FireModule(1024, 256, 512),\n\n            nn.MaxPool2d(kernel_size=2, stride=2, padding=0),\n            FireModule(1024, 512, 1024),\n            FireModule(2048, 512, 1024),\n            FireModule(2048, 512, 1024),\n\n            nn.Conv2d(2048, max_bbox_count*5*16, kernel_size=1, stride=1, padding='same')\n        )\n        \n    def forward(self, x):\n#         print('shape before first permute', x.shape)\n        x = x.permute((0, 3, 1, 2))\n#         print('shape after first permute', x.shape)\n        x = self.model(x)\n#         print('shape before second permute', x.shape)\n        x = x.permute((0, 2, 3, 1))\n#         print('shape after second permute', x.shape)\n        x = self.logits_layer(x)\n#         print('shape after logits layer', x.shape)\n        return x\n\n# Model initialization\nmodel = YoloModel()\n\n# # Initialize the model, loss function, and optimizer\n# model = SqueezeNet(3, max_bbox_count*16*5) # 3 - no of image color channels\n# x = x.to(memory_format=torch.channels_last)\nmodel = model.to(CONFIG['device'])","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:22.279788Z","iopub.execute_input":"2024-08-26T18:56:22.280563Z","iopub.status.idle":"2024-08-26T18:56:22.529028Z","shell.execute_reply.started":"2024-08-26T18:56:22.280532Z","shell.execute_reply":"2024-08-26T18:56:22.528271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.state_dict","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:22.530466Z","iopub.execute_input":"2024-08-26T18:56:22.530745Z","iopub.status.idle":"2024-08-26T18:56:22.535773Z","shell.execute_reply.started":"2024-08-26T18:56:22.530721Z","shell.execute_reply":"2024-08-26T18:56:22.535032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_params = sum(p.numel() for p in model.parameters())\n\nprint(total_params)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:22.679172Z","iopub.execute_input":"2024-08-26T18:56:22.679430Z","iopub.status.idle":"2024-08-26T18:56:22.684887Z","shell.execute_reply.started":"2024-08-26T18:56:22.679409Z","shell.execute_reply":"2024-08-26T18:56:22.683988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def criterion(y_pred, y_true):\n    return normalize_yolo_loss(y_pred, y_true)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:22.896929Z","iopub.execute_input":"2024-08-26T18:56:22.897225Z","iopub.status.idle":"2024-08-26T18:56:22.901422Z","shell.execute_reply.started":"2024-08-26T18:56:22.897202Z","shell.execute_reply":"2024-08-26T18:56:22.900479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.RMSprop(model.parameters(), lr=0.0001)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:51.961418Z","iopub.execute_input":"2024-08-26T18:56:51.961761Z","iopub.status.idle":"2024-08-26T18:56:51.967381Z","shell.execute_reply.started":"2024-08-26T18:56:51.961736Z","shell.execute_reply":"2024-08-26T18:56:51.966412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epoch(model, optimizer, dataloader, device, epoch, num_epochs):\n    model.train()\n    running_loss = 0.0\n    train_loss = 0.0\n    epoch_done = epoch\n    batch_size = 16\n    num_samples_per_epoch = (len(dataloader)//batch_size) * batch_size\n    \n    batch_size = CONFIG['train_batch_size']\n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n\n    for train_step, data in bar:\n        images = data['image'].to(device, dtype=torch.float)\n        labels = data['label'].to(device, dtype=torch.float)\n        \n        outputs = model(images)\n\n        loss = criterion(outputs, labels)\n        \n        optimizer.zero_grad()\n\n        loss.backward()\n        \n        optimizer.step()\n        \n        running_loss += loss.item()\n\n        train_loss = running_loss / (train_step+1)\n                \n        bar.set_postfix(epoch=f\"{epoch}/{num_epochs}\", loss ='{: .4f}'.format(train_loss), learning_rate='{: .4f}'.format(optimizer.param_groups[0]['lr']))\n    \n    return train_loss","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:52.878758Z","iopub.execute_input":"2024-08-26T18:56:52.879497Z","iopub.status.idle":"2024-08-26T18:56:52.887832Z","shell.execute_reply.started":"2024-08-26T18:56:52.879464Z","shell.execute_reply":"2024-08-26T18:56:52.886825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(model, optimizer, device, num_epochs, scheduler):\n    if torch.cuda.is_available():\n        print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n    start = time.time()\n    history = defaultdict(list)\n    \n    for epoch in range(1, num_epochs + 1):\n        \n        epoch_loss = train_epoch(model=model, optimizer=optimizer, dataloader=train_dataloader, \n                                      device=CONFIG['device'], epoch=epoch, \n                                      num_epochs=num_epochs)\n        \n        history['train_loss'].append(epoch_loss)\n#         bar.set_postfix(epoch=f\"{epoch}/{num_epochs}\", loss ='{: .4f}'.format(train_loss),\n#                         learning_rate='{:.4f}'.format(optimizer.param_groups[0]['lr']))\n    \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    \n    return history","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:53.246534Z","iopub.execute_input":"2024-08-26T18:56:53.246879Z","iopub.status.idle":"2024-08-26T18:56:53.254348Z","shell.execute_reply.started":"2024-08-26T18:56:53.246850Z","shell.execute_reply":"2024-08-26T18:56:53.253364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.state_dict","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:53.790788Z","iopub.execute_input":"2024-08-26T18:56:53.791239Z","iopub.status.idle":"2024-08-26T18:56:53.795287Z","shell.execute_reply.started":"2024-08-26T18:56:53.791207Z","shell.execute_reply":"2024-08-26T18:56:53.794300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optimizer","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:54.180369Z","iopub.execute_input":"2024-08-26T18:56:54.180683Z","iopub.status.idle":"2024-08-26T18:56:54.186740Z","shell.execute_reply.started":"2024-08-26T18:56:54.180657Z","shell.execute_reply":"2024-08-26T18:56:54.185960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = run_training(model=model, optimizer=optimizer,\n                              device=CONFIG['device'],\n                              num_epochs=500, scheduler=None)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:56:54.547169Z","iopub.execute_input":"2024-08-26T18:56:54.547466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), '/kaggle/working/latest_yolo_model.pth')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_len = len(history[\"train_loss\"])","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:48:31.139848Z","iopub.status.idle":"2024-08-26T18:48:31.140314Z","shell.execute_reply.started":"2024-08-26T18:48:31.140084Z","shell.execute_reply":"2024-08-26T18:48:31.140103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(range(history_len), history[\"train_loss\"], label=\"Train Loss\")\n# plt.plot(range(history_len), history[\"learning_rate\"], label=\"Learning Rate\")\nplt.xlabel(\"epochs\")\n# plt.ylabel(\"Loss\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:21:09.691084Z","iopub.execute_input":"2024-08-26T18:21:09.691354Z","iopub.status.idle":"2024-08-26T18:21:09.951593Z","shell.execute_reply.started":"2024-08-26T18:21:09.691331Z","shell.execute_reply":"2024-08-26T18:21:09.950620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get and resize test images\nsizes_test = []\nprint('resize test images ... ')\nsys.stdout.flush()\n\nfor n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):\n    path = TEST_PATH + \"/\" + id_\n    img = imread(path + '/images/' + id_ + '.png')[:, :, :3]\n    sizes_test.append([img.shape[0], img.shape[1]])\n    img = resize(img, (img_height, img_width), mode='constant', preserve_range=True)\n    test_images[n] = img","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:21:09.968054Z","iopub.execute_input":"2024-08-26T18:21:09.968392Z","iopub.status.idle":"2024-08-26T18:21:13.001072Z","shell.execute_reply.started":"2024-08-26T18:21:09.968361Z","shell.execute_reply":"2024-08-26T18:21:13.000149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img = test_images[1].copy()\ntest_img = torch.from_numpy(test_img).float().to(CONFIG['device'])\ntest_img = torch.unsqueeze(test_img, dim = 0)\n# print(test_img.shape)\nmodel.eval()\nwith torch.inference_mode():\n    result = model(test_img)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:23:17.830557Z","iopub.execute_input":"2024-08-26T18:23:17.830938Z","iopub.status.idle":"2024-08-26T18:23:17.845479Z","shell.execute_reply.started":"2024-08-26T18:23:17.830908Z","shell.execute_reply":"2024-08-26T18:23:17.844597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_img = test_img\npred_boxes = result[0].cpu()\ntest_img = test_img.cpu().numpy()\n# print(type(test_img))\ndraw_data(test_img.squeeze()/255., pred_boxes, thresh=0.1)","metadata":{"execution":{"iopub.status.busy":"2024-08-26T18:23:18.258743Z","iopub.execute_input":"2024-08-26T18:23:18.259621Z","iopub.status.idle":"2024-08-26T18:23:18.698180Z","shell.execute_reply.started":"2024-08-26T18:23:18.259588Z","shell.execute_reply":"2024-08-26T18:23:18.697223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}