{"metadata":{"colab":{"provenance":[],"gpuType":"T4"},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"accelerator":"GPU","kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":19989,"databundleVersionId":1160143,"sourceType":"competition"},{"sourceId":7843326,"sourceType":"datasetVersion","datasetId":4598498}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# pip install google-colab\n","metadata":{"id":"SWfzs_8G6kyB","outputId":"a1a5e036-b0d2-4cc4-bd34-ed65c67e0289","execution":{"iopub.status.busy":"2024-04-03T19:01:45.188464Z","iopub.execute_input":"2024-04-03T19:01:45.188735Z","iopub.status.idle":"2024-04-03T19:01:45.193303Z","shell.execute_reply.started":"2024-04-03T19:01:45.188709Z","shell.execute_reply":"2024-04-03T19:01:45.192373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from google.colab import drive\n# drive.mount('/content/data',force_remount=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T19:01:45.194536Z","iopub.execute_input":"2024-04-03T19:01:45.195235Z","iopub.status.idle":"2024-04-03T19:01:45.202899Z","shell.execute_reply.started":"2024-04-03T19:01:45.195204Z","shell.execute_reply":"2024-04-03T19:01:45.202217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pycocotools","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:07.695005Z","iopub.execute_input":"2024-04-03T21:36:07.695710Z","iopub.status.idle":"2024-04-03T21:36:23.093942Z","shell.execute_reply.started":"2024-04-03T21:36:07.695674Z","shell.execute_reply":"2024-04-03T21:36:23.092881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nimport torch\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision.datasets import CocoDetection ,wrap_dataset_for_transforms_v2\nfrom pycocotools.coco import COCO\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nfrom tqdm import tqdm","metadata":{"id":"ZoXXfQ0AKA6U","outputId":"8a789c57-cf03-4c3b-d1b5-3b6b41d568fb","execution":{"iopub.status.busy":"2024-04-03T21:36:23.095896Z","iopub.execute_input":"2024-04-03T21:36:23.096215Z","iopub.status.idle":"2024-04-03T21:36:34.123823Z","shell.execute_reply.started":"2024-04-03T21:36:23.096185Z","shell.execute_reply":"2024-04-03T21:36:34.122891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TODO: crop each image into two pages","metadata":{"id":"keHa68idVbR-"}},{"cell_type":"markdown","source":"# CONSTENTS","metadata":{"id":"OvSl8UZiaQBx"}},{"cell_type":"code","source":"ann_path = '/kaggle/input/historical/GP/Data/dataset_coco2.json'\nroot = '/kaggle/input/historical/GP/Data/imgs_png/asmaa_nawawi as PNG'\n\nBATCH_SIZE = 4\nLR = 1e-2\nEPOCHS = 8\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\n## more on constents will be added soon","metadata":{"id":"XGAP8NppaW_w","execution":{"iopub.status.busy":"2024-04-03T21:36:34.125284Z","iopub.execute_input":"2024-04-03T21:36:34.125661Z","iopub.status.idle":"2024-04-03T21:36:34.182866Z","shell.execute_reply.started":"2024-04-03T21:36:34.125635Z","shell.execute_reply":"2024-04-03T21:36:34.181949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Loading\n\n","metadata":{"id":"C64BMaVGTfx0"}},{"cell_type":"code","source":"import json\nf = open(ann_path)\njs = json.load(f)\njs.keys()","metadata":{"id":"yy5OGAmDLpwj","outputId":"d65cfcec-bfb8-413f-ba71-f725ec2a96ec","execution":{"iopub.status.busy":"2024-04-03T21:36:34.184967Z","iopub.execute_input":"2024-04-03T21:36:34.185253Z","iopub.status.idle":"2024-04-03T21:36:34.260643Z","shell.execute_reply.started":"2024-04-03T21:36:34.185228Z","shell.execute_reply":"2024-04-03T21:36:34.259808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"se-qPh0TWkhw","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = CocoDetection(annFile=ann_path,root=root)\nsample = dataset[1]\nbboxes = []\nfor a in sample[1]:\n    bboxes.append(a['bbox'])\nsample[0].size","metadata":{"id":"xClwMd25XFpZ","outputId":"36dc90ca-6ed7-4619-b0ef-2190fd7a0db8","execution":{"iopub.status.busy":"2024-04-03T21:36:34.261718Z","iopub.execute_input":"2024-04-03T21:36:34.261981Z","iopub.status.idle":"2024-04-03T21:36:34.868266Z","shell.execute_reply.started":"2024-04-03T21:36:34.261957Z","shell.execute_reply":"2024-04-03T21:36:34.867436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:54:48.595544Z","iopub.execute_input":"2024-04-03T21:54:48.596274Z","iopub.status.idle":"2024-04-03T21:54:48.600220Z","shell.execute_reply.started":"2024-04-03T21:54:48.596242Z","shell.execute_reply":"2024-04-03T21:54:48.599307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visual some images with its bboxes\n\n# plt.figure()\nplt.imshow(sample[0])\nfor box in bboxes:\n\n    x, y, width, height = box\n    # width = (x2-x)\n    # height = (y2-y)\n\n    plt.gca().add_patch(patches.Rectangle((x,y),width,height,\n                    edgecolor='red',\n                    facecolor='none',\n                    lw=1))\n\n","metadata":{"id":"SZIVbNS0Wgvn","outputId":"1b8cecbb-c32d-4374-c1cf-ec87172914d1","execution":{"iopub.status.busy":"2024-04-03T21:36:34.869466Z","iopub.execute_input":"2024-04-03T21:36:34.869755Z","iopub.status.idle":"2024-04-03T21:36:40.449590Z","shell.execute_reply.started":"2024-04-03T21:36:34.869730Z","shell.execute_reply":"2024-04-03T21:36:40.448742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset and DataLoader","metadata":{"id":"QUybQqK5Tne6"}},{"cell_type":"code","source":"# Abdulshakoor:\n# =================================================================================================\n# input: coco dataset                # it's already defined\n# returned value: new_dataset        # as a list of tuples(image,target_dims)  ==> [(image,[target_dims]), (image,[target_dims]), .... ,(image,[target_dims])]\n# NOTE: target_dims = [xmin, ymin, w, h]\n\n# new_dataset  [21]           [1]             [0]                [0]\n# new_dataset [page] [0:image or 1:target] [n_target] [dims i.e. (xmin, ymin, w, h)]\n# =================================================================================================\n\n# Mohammed :\n# ===========================================\n# the normlization part would probably be removed\nimport copy\n\ndef split_pages(dataset):\n  copy_dataset = copy.deepcopy(dataset)\n  new_dataset = []  \n  # splitting loop:\n  for sample in copy_dataset:\n    WIDTH, HIEGHT = sample[0].size\n    \n    # split pages                               image,     ymin,        xmin,           h,             w\n    left_page  = transforms.functional.crop(img=sample[0], top=0,      left=0,     height=HIEGHT, width=WIDTH//2)\n    right_page = transforms.functional.crop(img=sample[0], top=0, left=WIDTH//2+1, height=HIEGHT, width=WIDTH//2)\n\n    # split bboxes (targets) loop\n    right_page_labels = {'boxes':[]}\n    left_page_labels = {'boxes':[]}\n\n    for label in sample[1]:\n      if (label['bbox'][0] >= (WIDTH//2)) :                 # if xmin >= mid_page: it's right_page\n        label['bbox'][0] = label['bbox'][0] - (WIDTH/2)    # update xmin in right_pages\n        # covert it to voc format: xmin,ymin,xmax,ymax\n        # and normlize it [0,1]\n        xmin, ymin, width, height = label['bbox']\n        label['bbox'] = torch.tensor([xmin/WIDTH,ymin/HIEGHT,(xmin+width)/WIDTH,(ymin+height)/HIEGHT])\n        right_page_labels['boxes'].append(label['bbox'])\n      else :\n        # covert it to voc format: xmin,ymin,xmax,ymax\n        # and normlize it [0,1]        \n        xmin, ymin, width, height = label['bbox']\n        label['bbox'] = torch.tensor([xmin/WIDTH,ymin/HIEGHT,(xmin+width)/WIDTH,(ymin+height)/HIEGHT])\n        left_page_labels['boxes'].append(label['bbox'])\n\n    # store images & bboxes in order (arabic text starts from right to left)\n    right_page_labels['labels'] = torch.ones(len(right_page_labels['boxes']),dtype=torch.int64).to(device)\n    left_page_labels['labels'] = torch.ones(len(left_page_labels['boxes']),dtype=torch.int64).to(device)\n    \n    right_page_labels['boxes'] = torch.stack(right_page_labels['boxes'])\n    left_page_labels['boxes'] = torch.stack(left_page_labels['boxes'])\n    \n    right_sample = [right_page,right_page_labels]\n    left_sample = [left_page,left_page_labels]\n    new_dataset.append(right_sample)\n    new_dataset.append(left_sample)\n\n\n  return new_dataset\n\n\ndataset = split_pages(dataset)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:40.450678Z","iopub.execute_input":"2024-04-03T21:36:40.450951Z","iopub.status.idle":"2024-04-03T21:36:46.803466Z","shell.execute_reply.started":"2024-04-03T21:36:40.450927Z","shell.execute_reply":"2024-04-03T21:36:46.802683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset[0][0].size","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:46.804615Z","iopub.execute_input":"2024-04-03T21:36:46.804917Z","iopub.status.idle":"2024-04-03T21:36:46.810887Z","shell.execute_reply.started":"2024-04-03T21:36:46.804891Z","shell.execute_reply":"2024-04-03T21:36:46.809956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imgs = torch.stack([torch.tensor(np.array(image)/255,dtype=torch.float64) for image,target in dataset],dim=0)\n# imgs = imgs.permute(3,1,2,0)\n# mean = imgs.reshape(3,-1).mean(dim=1)\n# std = imgs.reshape(3,-1).std(dim=1)\n\n# print(mean)\n# print(std)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:46.812067Z","iopub.execute_input":"2024-04-03T21:36:46.812331Z","iopub.status.idle":"2024-04-03T21:36:46.827725Z","shell.execute_reply.started":"2024-04-03T21:36:46.812308Z","shell.execute_reply":"2024-04-03T21:36:46.826823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mean: tensor([0.5381, 0.6001, 0.4409], dtype=torch.float64)\n# std: tensor([0.1344, 0.1472, 0.1295], dtype=torch.float64)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:46.830849Z","iopub.execute_input":"2024-04-03T21:36:46.831200Z","iopub.status.idle":"2024-04-03T21:36:46.838768Z","shell.execute_reply.started":"2024-04-03T21:36:46.831175Z","shell.execute_reply":"2024-04-03T21:36:46.838117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visual some images with its bboxes\nsample = dataset[3]\n# plt.figure()\nplt.imshow(sample[0])\nfor box in sample[1]['boxes']:\n    \n    x, y, x2, y2 = box\n    width = (x2-x)\n    height = (y2-y)\n\n    plt.gca().add_patch(patches.Rectangle((x*5689,y*4267),width*5689,height*4267,\n                    edgecolor='red',\n                    facecolor='none',\n                    lw=1))\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:46.839749Z","iopub.execute_input":"2024-04-03T21:36:46.840067Z","iopub.status.idle":"2024-04-03T21:36:49.805276Z","shell.execute_reply.started":"2024-04-03T21:36:46.840036Z","shell.execute_reply":"2024-04-03T21:36:49.804364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(dataset[0][1]['boxes'])\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:49.806458Z","iopub.execute_input":"2024-04-03T21:36:49.806754Z","iopub.status.idle":"2024-04-03T21:36:49.812626Z","shell.execute_reply.started":"2024-04-03T21:36:49.806730Z","shell.execute_reply":"2024-04-03T21:36:49.811692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\nclass My_Dataset(Dataset):\n    def __init__(self,dataset,trans = None):\n        self.dataset = dataset\n        self.trans = trans\n        \n    def __len__(self):\n        return len(self.dataset)\n    \n    \n    def __getitem__(self,idx):\n        image , target = self.dataset[idx]\n        if self.trans:\n            #https://youtu.be/rAdLwKJBvPM?feature=shared&t=1649\n            agumentation = self.trans(image=np.array(image),\n                                      bboxes=target['boxes'])\n            image = agumentation['image']\n            target['boxes'] = agumentation['bboxes']\n        \n        return image,target\n            \n            \n\n            \n        \n        ","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:49.813766Z","iopub.execute_input":"2024-04-03T21:36:49.814039Z","iopub.status.idle":"2024-04-03T21:36:49.823740Z","shell.execute_reply.started":"2024-04-03T21:36:49.814016Z","shell.execute_reply":"2024-04-03T21:36:49.822882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport cv2\n\ntrans = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.RandomBrightnessContrast(p=0.2),\n    A.SafeRotate(limit=(-15,15)),\n    A.Normalize(mean=[0.5381, 0.6001, 0.4409],std=[0.1344, 0.1472, 0.1295]),\n    ToTensorV2()\n], bbox_params=A.BboxParams(format='pascal_voc',label_fields=[]) )\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:49.824744Z","iopub.execute_input":"2024-04-03T21:36:49.824990Z","iopub.status.idle":"2024-04-03T21:36:52.969142Z","shell.execute_reply.started":"2024-04-03T21:36:49.824969Z","shell.execute_reply":"2024-04-03T21:36:52.968360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_dataset = My_Dataset(dataset,trans)\n# my_dataset[3][1]['boxes']","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:52.970232Z","iopub.execute_input":"2024-04-03T21:36:52.970736Z","iopub.status.idle":"2024-04-03T21:36:52.975052Z","shell.execute_reply.started":"2024-04-03T21:36:52.970702Z","shell.execute_reply":"2024-04-03T21:36:52.974156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(my_dataset[3][1]['boxes'])","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:52.976128Z","iopub.execute_input":"2024-04-03T21:36:52.976431Z","iopub.status.idle":"2024-04-03T21:36:53.398184Z","shell.execute_reply.started":"2024-04-03T21:36:52.976384Z","shell.execute_reply":"2024-04-03T21:36:53.397219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n\nindices = np.arange(len(my_dataset))\n\n# np.random.shuffle(indices) We may shuffle it later\n\n# Split indices\nsplit_point = int(0.8*len(my_dataset))\ntrain_indices , val_indices = indices[:split_point],indices[split_point:]\n\n# Create train set and val set\ntrain_dataset = torch.utils.data.Subset(my_dataset,train_indices)\nval_dataset = torch.utils.data.Subset(my_dataset,val_indices)\n\ntrain_dataset[2][0]","metadata":{"id":"x8B3LxfjYf2H","outputId":"31ccb5aa-493a-4db4-ed34-d8ab5991b410","execution":{"iopub.status.busy":"2024-04-03T21:36:53.399480Z","iopub.execute_input":"2024-04-03T21:36:53.400196Z","iopub.status.idle":"2024-04-03T21:36:53.764009Z","shell.execute_reply.started":"2024-04-03T21:36:53.400158Z","shell.execute_reply":"2024-04-03T21:36:53.763090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(train_dataset[0][1]['boxes'])","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:53.765497Z","iopub.execute_input":"2024-04-03T21:36:53.765845Z","iopub.status.idle":"2024-04-03T21:36:54.048429Z","shell.execute_reply.started":"2024-04-03T21:36:53.765812Z","shell.execute_reply":"2024-04-03T21:36:54.047548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fun(batch):\n    # list[ pair(imgages,labels) ]\n#     print(type(batch[0][1]['boxes'])) # list\n#     return 0\n    targets = [{}]\n    imgs = []\n    for sample in batch:\n        all_boxes = []\n        imgs.append(sample[0])\n        boxes = sample[1]['boxes']\n        for box in boxes:\n            all_boxes.append(torch.tensor(box).to(device))\n                \n        all_boxes = torch.stack(all_boxes)\n        tar = {'boxes':all_boxes,'labels':sample[1]['labels']}\n        targets.append(tar)\n        \n    targets.pop(0)\n    \n        \n#         targets[]        \n#         batch[i][1]['boxes'] = torch.tensor(sample[1]['boxes'])\n#     boxes = torch.stack(boxes)    \n    return [imgs,targets]","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:54.049523Z","iopub.execute_input":"2024-04-03T21:36:54.049801Z","iopub.status.idle":"2024-04-03T21:36:54.056213Z","shell.execute_reply.started":"2024-04-03T21:36:54.049778Z","shell.execute_reply":"2024-04-03T21:36:54.055232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We didn't use the default collate_fn because the number of boxes on each image are not the same.\n# the default one is torch.stack, and we cannot stack tensors with different shape\ntrain_loader = DataLoader(train_dataset,batch_size=BATCH_SIZE, collate_fn= collate_fun)\n# train_loader = DataLoader(train_dataset,batch_size=BATCH_SIZE, collate_fn= lambda x: list(zip(*x)))\nval_loader = DataLoader(val_dataset,batch_size=BATCH_SIZE, collate_fn=collate_fun )\nimages,targets = next(iter(train_loader))\nlen(targets)\n","metadata":{"id":"wN6q8zN7pwg8","outputId":"f1afeae8-e303-474a-ce45-ad4b626fc03e","execution":{"iopub.status.busy":"2024-04-03T21:36:54.057266Z","iopub.execute_input":"2024-04-03T21:36:54.057525Z","iopub.status.idle":"2024-04-03T21:36:55.187938Z","shell.execute_reply.started":"2024-04-03T21:36:54.057502Z","shell.execute_reply":"2024-04-03T21:36:55.186827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images,targets = next(iter(train_loader))\n# targets","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:36:55.189120Z","iopub.execute_input":"2024-04-03T21:36:55.189465Z","iopub.status.idle":"2024-04-03T21:36:56.142516Z","shell.execute_reply.started":"2024-04-03T21:36:55.189435Z","shell.execute_reply":"2024-04-03T21:36:56.141722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for example: 15, 11, 10, 10 , 10\n# 5,10,4","metadata":{"id":"9cNYjoB-SgvJ","execution":{"iopub.status.busy":"2024-04-03T19:02:19.093185Z","iopub.execute_input":"2024-04-03T19:02:19.093602Z","iopub.status.idle":"2024-04-03T19:02:19.098115Z","shell.execute_reply.started":"2024-04-03T19:02:19.093566Z","shell.execute_reply":"2024-04-03T19:02:19.097204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"id":"ugv3qE06pUAG"}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# targets[0][0]['bbox']\n","metadata":{"id":"lEx3HC_Iy_gw","outputId":"fceff69d-3b37-4aa7-d8b3-a602e76187b0","execution":{"iopub.status.busy":"2024-04-03T19:02:19.099166Z","iopub.execute_input":"2024-04-03T19:02:19.099507Z","iopub.status.idle":"2024-04-03T19:02:19.110943Z","shell.execute_reply.started":"2024-04-03T19:02:19.099481Z","shell.execute_reply":"2024-04-03T19:02:19.110039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision\nimport torchvision.models.detection as models\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nimport torchvision.models.detection.faster_rcnn as FasterRCNN\n\nweights = FasterRCNN.FasterRCNN_ResNet50_FPN_V2_Weights.DEFAULT\n\nmodel = models.fasterrcnn_resnet50_fpn_v2(weights=weights, box_score_thresh=0.5)\n\n# model = torchvision.models.detection.fasterrcnn_mobilenet_v3_large_fpn(weights=FasterRCNN.FasterRCNN_MobileNet_V3_Large_FPN_Weights.DEFAULT)\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nnum_classes = 2  # background + words\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n# # Modify the cls_logits layer\n# num_rpn_logits = 3\n# model.rpn.head.cls_logits = nn.Conv2d(256, num_rpn_logits, kernel_size=(1, 1), stride=(1, 1))\n\n# # Modify the bbox_pred layer\n# model.rpn.head.bbox_pred = nn.Conv2d(256, num_rpn_logits*4, kernel_size=(1, 1), stride=(1, 1))\n\nmodel=model.to(device)\n\n","metadata":{"id":"HtgxyiAMr-n6","outputId":"5dd906cd-e1da-44ef-cb4f-9cbb965777e2","execution":{"iopub.status.busy":"2024-04-03T21:53:32.280089Z","iopub.execute_input":"2024-04-03T21:53:32.280460Z","iopub.status.idle":"2024-04-03T21:53:33.173935Z","shell.execute_reply.started":"2024-04-03T21:53:32.280426Z","shell.execute_reply":"2024-04-03T21:53:33.172980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Freeze all layers except the last two\nnum_layers = len(model.backbone.body)\nfor i, param in enumerate(model.backbone.body.parameters()):\n    if i < num_layers - 1:\n        param.requires_grad = False\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:53:34.404906Z","iopub.execute_input":"2024-04-03T21:53:34.405537Z","iopub.status.idle":"2024-04-03T21:53:34.411435Z","shell.execute_reply.started":"2024-04-03T21:53:34.405503Z","shell.execute_reply":"2024-04-03T21:53:34.410439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = [p for p in model.parameters() if p.requires_grad]\n\noptimizer = torch.optim.Adam(params, lr=1e-4, weight_decay=0.0001)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:53:38.414232Z","iopub.execute_input":"2024-04-03T21:53:38.415167Z","iopub.status.idle":"2024-04-03T21:53:38.421180Z","shell.execute_reply.started":"2024-04-03T21:53:38.415129Z","shell.execute_reply":"2024-04-03T21:53:38.420415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:52:26.773187Z","iopub.execute_input":"2024-04-03T21:52:26.774080Z","iopub.status.idle":"2024-04-03T21:52:26.779693Z","shell.execute_reply.started":"2024-04-03T21:52:26.774042Z","shell.execute_reply":"2024-04-03T21:52:26.778671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, param in enumerate(model.backbone.body.parameters()):\n    print(param.requires_grad)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:52:28.322246Z","iopub.execute_input":"2024-04-03T21:52:28.322616Z","iopub.status.idle":"2024-04-03T21:52:28.329980Z","shell.execute_reply.started":"2024-04-03T21:52:28.322585Z","shell.execute_reply":"2024-04-03T21:52:28.329132Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(\n    optimizer,\n    gamma=0.7\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:46:30.176962Z","iopub.execute_input":"2024-04-03T21:46:30.177642Z","iopub.status.idle":"2024-04-03T21:46:30.188706Z","shell.execute_reply.started":"2024-04-03T21:46:30.177587Z","shell.execute_reply":"2024-04-03T21:46:30.186923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.rpn","metadata":{"execution":{"iopub.status.busy":"2024-04-03T19:02:20.079315Z","iopub.execute_input":"2024-04-03T19:02:20.079687Z","iopub.status.idle":"2024-04-03T19:02:20.092769Z","shell.execute_reply.started":"2024-04-03T19:02:20.079653Z","shell.execute_reply":"2024-04-03T19:02:20.091665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\ndef set_seed(seed):\n    \"\"\"Set seed\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n    print(f'Set seed {seed}')\n\nset_seed(42)  \n\n","metadata":{"id":"2vd24-O909ql","execution":{"iopub.status.busy":"2024-04-03T21:53:44.537624Z","iopub.execute_input":"2024-04-03T21:53:44.537978Z","iopub.status.idle":"2024-04-03T21:53:44.544809Z","shell.execute_reply.started":"2024-04-03T21:53:44.537949Z","shell.execute_reply":"2024-04-03T21:53:44.543710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# targets needed to ba as a list of list\n# each list has some amount of boxes and its crossponding labels as a dict\n\n# ttt is just for debuge purpose\n# ttt = [{}]\nset_seed(42)\n\nls = []\nls_val = []\nfor epoch in range(4):\n    model.train()\n    train_losses = 0\n    val_losses = 0\n    for images, targets in tqdm(train_loader):\n        images = list(image.to(device) for image in images)\n        loss_dict = model(images,targets)\n        losses = sum(loss for loss in loss_dict.values())\n        optimizer.zero_grad()        \n        losses.backward()\n        optimizer.step()\n\n        train_losses += losses\n\n    with torch.inference_mode():\n        for images, targets in tqdm(val_loader):\n            images = list(image.to(device) for image in images)\n            losses = model(images,targets)\n            losses = sum(loss for loss in loss_dict.values())\n            val_losses += losses\n\n    ls.append(train_losses.item()/len(train_loader))\n    ls_val.append(val_losses.item()/len(val_loader))\n    print(f\"Epoch:{epoch}, Train_loss: {train_losses.item()/len(train_loader)} , val_loss:{val_losses.item()/len(val_loader)} \")\n    lr_scheduler.step()\n    print(lr_scheduler.get_last_lr())\n\n\n# Epoch:8, Train_loss: 2.349443700828519 , val_loss:0.9182189203510349 \n","metadata":{"id":"xAV-0w3mhcJj","execution":{"iopub.status.busy":"2024-04-03T21:53:46.254368Z","iopub.execute_input":"2024-04-03T21:53:46.254767Z","iopub.status.idle":"2024-04-03T21:53:48.260981Z","shell.execute_reply.started":"2024-04-03T21:53:46.254734Z","shell.execute_reply":"2024-04-03T21:53:48.259506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(ls,label=\"Train loss\")\nplt.plot(ls_val,label=\"Val loss\")\n\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Total losses\")\n\nplt.legend()\nplt.show()","metadata":{"id":"5sS0o-tz7RaW","execution":{"iopub.status.busy":"2024-04-03T21:47:30.175665Z","iopub.execute_input":"2024-04-03T21:47:30.175992Z","iopub.status.idle":"2024-04-03T21:47:30.447782Z","shell.execute_reply.started":"2024-04-03T21:47:30.175967Z","shell.execute_reply":"2024-04-03T21:47:30.446872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def view(images,labels,k,std=1,mean=0):\n    figure = plt.figure(figsize=(30,30))\n    images=list(images)\n    labels=list(labels)\n    for i in range(k):\n        out=torchvision.utils.make_grid(images[i])\n        inp=out.cpu().numpy().transpose((1,2,0))\n        inp=np.array(std)*inp+np.array(mean)\n        inp=np.clip(inp,0,1)\n        ax = figure.add_subplot(2,2, i + 1)\n        ax.imshow(images[i].cpu().detach().numpy().transpose((1,2,0)))\n        l=labels[i]['boxes'].cpu().detach().numpy()\n        l[:,2]=l[:,2]-l[:,0]\n        l[:,3]=l[:,3]-l[:,1]\n        for j in range(len(l)):\n            ax.add_patch(patches.Rectangle((l[j][0],l[j][1]),l[j][2],l[j][3],linewidth=2,edgecolor='w',facecolor='none'))\n\n# view(images,labels,4)","metadata":{"id":"Dio-U37t3sTc","execution":{"iopub.status.busy":"2024-04-03T21:47:37.284137Z","iopub.execute_input":"2024-04-03T21:47:37.284545Z","iopub.status.idle":"2024-04-03T21:47:37.294129Z","shell.execute_reply.started":"2024-04-03T21:47:37.284504Z","shell.execute_reply":"2024-04-03T21:47:37.293140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outtt = [{}]\n\nmodel.eval()\nfor images, targets in val_loader:\n    images = list(image.to(device) for image in images)\n    out_dict = model(images)\n    outtt  = out_dict\n    view(images,out_dict,1)\n    break\n\n\n\n","metadata":{"id":"lLM05DcsaAMG","execution":{"iopub.status.busy":"2024-04-03T21:47:37.664690Z","iopub.execute_input":"2024-04-03T21:47:37.664959Z","iopub.status.idle":"2024-04-03T21:47:43.796961Z","shell.execute_reply.started":"2024-04-03T21:47:37.664937Z","shell.execute_reply":"2024-04-03T21:47:43.795926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(outtt[0]['boxes'])","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:47:43.798459Z","iopub.execute_input":"2024-04-03T21:47:43.798794Z","iopub.status.idle":"2024-04-03T21:47:43.805967Z","shell.execute_reply.started":"2024-04-03T21:47:43.798765Z","shell.execute_reply":"2024-04-03T21:47:43.804751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view(images,outtt,1)\n","metadata":{"id":"0eqsgBCqoago","execution":{"iopub.status.busy":"2024-04-03T19:03:11.325970Z","iopub.status.idle":"2024-04-03T19:03:11.326516Z","shell.execute_reply.started":"2024-04-03T19:03:11.326231Z","shell.execute_reply":"2024-04-03T19:03:11.326253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outtt[0]","metadata":{"execution":{"iopub.status.busy":"2024-04-03T19:03:11.327648Z","iopub.status.idle":"2024-04-03T19:03:11.328172Z","shell.execute_reply.started":"2024-04-03T19:03:11.327906Z","shell.execute_reply":"2024-04-03T19:03:11.327928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torchvision.datasets import CocoDetection\n\n# class CustomCocoDetection(CocoDetection):\n#     def __getitem__(self, idx):\n#         # Get the original image and target\n#         img, target = super().__getitem__(idx)\n\n#         # Get the width and height of the image\n#         width, height = img.size\n\n#         # Normalize the bounding boxes\n#         for box in target['boxes']:\n#             box[0] /= width  # Normalize x_min\n#             box[1] /= height  # Normalize y_min\n#             box[2] /= width  # Normalize box width\n#             box[3] /= height  # Normalize box height\n\n#         return img, target\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T19:03:11.329912Z","iopub.status.idle":"2024-04-03T19:03:11.330424Z","shell.execute_reply.started":"2024-04-03T19:03:11.330147Z","shell.execute_reply":"2024-04-03T19:03:11.330167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# هبد","metadata":{"id":"urAPIUvGwk-o"}},{"cell_type":"code","source":"# !git clone https://github.com/ying09/TextFuseNet.git\n# !unzip TextFuseNet-master\n# %cd TextFuseNet-master\n# !pip install fvcore-master.zip\n# !python setup.py build develop\n","metadata":{"id":"kTdyLQZkw21N","execution":{"iopub.status.busy":"2024-04-03T19:03:11.331934Z","iopub.status.idle":"2024-04-03T19:03:11.332312Z","shell.execute_reply.started":"2024-04-03T19:03:11.332123Z","shell.execute_reply":"2024-04-03T19:03:11.332138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cd TextFuseNet-master\n# ! python /content/TextFuseNet/setup.py build develop","metadata":{"id":"FuiDn-EJyDvt","execution":{"iopub.status.busy":"2024-04-03T19:03:11.333696Z","iopub.status.idle":"2024-04-03T19:03:11.334063Z","shell.execute_reply.started":"2024-04-03T19:03:11.333882Z","shell.execute_reply":"2024-04-03T19:03:11.333898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# an easy way for drawing bboxes\nfrom torchvision.ops import box_convert,nms\nfrom torchvision.utils import draw_bounding_boxes\nfrom torchvision.transforms.functional import pil_to_tensor, to_pil_image\n\nnms(outtt[2]['boxes'],outtt[2]['scores'],0.9)\n\nto_pil_image(\n    draw_bounding_boxes(\n        images[0].to(torch.uint8),\n        outtt[0]['boxes'].to(torch.int32),\n        colors=\"red\",\n#         labels=labels,\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:50:58.769619Z","iopub.execute_input":"2024-04-03T21:50:58.770316Z","iopub.status.idle":"2024-04-03T21:51:00.092507Z","shell.execute_reply.started":"2024-04-03T21:50:58.770283Z","shell.execute_reply":"2024-04-03T21:51:00.091591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outtt['boxes']","metadata":{"execution":{"iopub.status.busy":"2024-04-03T21:49:55.893245Z","iopub.execute_input":"2024-04-03T21:49:55.893966Z","iopub.status.idle":"2024-04-03T21:49:55.924237Z","shell.execute_reply.started":"2024-04-03T21:49:55.893931Z","shell.execute_reply":"2024-04-03T21:49:55.923192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# boxes_xyxy\nouttt\n# outtt[1]['boxes']","metadata":{"execution":{"iopub.status.busy":"2024-04-03T19:03:11.337772Z","iopub.status.idle":"2024-04-03T19:03:11.338163Z","shell.execute_reply.started":"2024-04-03T19:03:11.337974Z","shell.execute_reply":"2024-04-03T19:03:11.337989Z"},"trusted":true},"execution_count":null,"outputs":[]}]}