{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":43873,"databundleVersionId":5585780,"sourceType":"competition"},{"sourceId":7765706,"sourceType":"datasetVersion","datasetId":4542315}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install -q paddleocr\n# !pip install -q tight data prox dual common\n# !pip install -q paddle","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-05T14:15:46.444955Z","iopub.execute_input":"2024-03-05T14:15:46.445502Z","iopub.status.idle":"2024-03-05T14:15:46.478920Z","shell.execute_reply.started":"2024-03-05T14:15:46.445453Z","shell.execute_reply":"2024-03-05T14:15:46.477541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m pip install -q paddlepaddle==2.6.0 -i https://mirror.baidu.com/pypi/simple\n!pip install -q paddleocr","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:15:46.481033Z","iopub.execute_input":"2024-03-05T14:15:46.481873Z","iopub.status.idle":"2024-03-05T14:17:24.597999Z","shell.execute_reply.started":"2024-03-05T14:15:46.481829Z","shell.execute_reply":"2024-03-05T14:17:24.596178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import pipeline\n\nfix_spelling = pipeline(\"text2text-generation\",model=\"oliverguhr/spelling-correction-english-base\")\n\nprint(fix_spelling(\"lets do a comparsion\",max_length=2048))","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:17:24.600938Z","iopub.execute_input":"2024-03-05T14:17:24.601374Z","iopub.status.idle":"2024-03-05T14:18:00.293544Z","shell.execute_reply.started":"2024-03-05T14:17:24.601338Z","shell.execute_reply":"2024-03-05T14:18:00.291788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json \nimport os \n\npath = \"/kaggle/input/benetech-making-graphs-accessible/train/images\"\n  \n# cnt = 0\n# for i in os.scandir(path):\n#     if cnt>0:\n#         break\n#     cnt+=1\n#     with open(\"/kaggle/input/benetech-making-graphs-accessible/train/annotations/\"+i.name[:-3]+\"json\", 'r') as f:\n#         data = json.loads(f.read())\n#         print(data['text'],i.name)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:18:00.295355Z","iopub.execute_input":"2024-03-05T14:18:00.296301Z","iopub.status.idle":"2024-03-05T14:18:00.303881Z","shell.execute_reply.started":"2024-03-05T14:18:00.296259Z","shell.execute_reply":"2024-03-05T14:18:00.302980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from PIL import Image\n# from IPython.display import display\n# img = Image.open('/kaggle/input/benetech-making-graphs-accessible/train/images/0000ae6cbdb1.jpg')\n# display(img)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:18:00.306236Z","iopub.execute_input":"2024-03-05T14:18:00.306807Z","iopub.status.idle":"2024-03-05T14:18:00.319665Z","shell.execute_reply.started":"2024-03-05T14:18:00.306774Z","shell.execute_reply":"2024-03-05T14:18:00.318371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\n\n# sample_image = \"/kaggle/input/benetech-making-graphs-accessible/train/images/0000ae6cbdb1.jpg\"\n# image_cv = cv2.imread(sample_image)\n# img = cv2.cvtColor(image_cv, cv2.COLOR_BGR2RGB)\n# im_pil = Image.fromarray(img)\n# plt.imshow(image_cv)  \n# display(im_pil)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:18:00.321252Z","iopub.execute_input":"2024-03-05T14:18:00.321888Z","iopub.status.idle":"2024-03-05T14:18:00.424057Z","shell.execute_reply.started":"2024-03-05T14:18:00.321855Z","shell.execute_reply":"2024-03-05T14:18:00.422441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:18:00.425791Z","iopub.execute_input":"2024-03-05T14:18:00.426252Z","iopub.status.idle":"2024-03-05T14:18:00.434022Z","shell.execute_reply.started":"2024-03-05T14:18:00.426217Z","shell.execute_reply":"2024-03-05T14:18:00.431693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rotate_image(img, angle):\n    size_reverse = np.array(img.shape[1::-1]) # swap x with y\n    M = cv2.getRotationMatrix2D(tuple(size_reverse / 2.), angle, 1.)\n    MM = np.absolute(M[:,:2])\n    size_new = MM @ size_reverse\n    M[:,-1] += (size_new - size_reverse) / 2.\n    return cv2.warpAffine(img, M, tuple(size_new.astype(int)))\n\ndef matrix_image(img, angle):\n    size_reverse = np.array(img.shape[1::-1]) # swap x with y\n    M = cv2.getRotationMatrix2D(tuple(size_reverse / 2.), angle, 1.)\n    MM = np.absolute(M[:,:2])\n    size_new = MM @ size_reverse\n    M[:,-1] += (size_new - size_reverse) / 2.\n    return M\n\ndef rotate_vector(vector,matrix):\n    return [vector[0]*matrix[0][0]+vector[1]*matrix[0][1]+matrix[0][2],vector[0]*matrix[1][0]+vector[1]*matrix[1][1]+matrix[1][2]]\n\ndef invert_matrix(matrix):\n    m = matrix.copy()\n    det = m[0][0]*m[1][1]-m[0][1]*m[1][0]\n    m[0][0] = matrix[1][1]/det\n    m[0][1] = -matrix[0][1]/det\n    m[1][1] = matrix[0][0]/det\n    m[1][0] = -matrix[1][0]/det\n    m[0][2] = -matrix[0][2]*m[0][0]-matrix[1][2]*m[0][1]\n    m[1][2] = -matrix[0][2]*m[1][0]-matrix[1][2]*m[1][1]\n    return m\n\n# # test\n# DogImage = cv2.imread(\"/kaggle/input/benetech-making-graphs-accessible/train/images/0000ae6cbdb1.jpg\") \n# matrix = matrix_image(DogImage,-26)\n# print(rotate_vector(rotate_vector([0,1],matrix),invert_matrix(matrix)))\n  \n    \n# Modified = rotate_image(DogImage, 45) \n# plt.imshow(Modified)  ","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:18:00.436633Z","iopub.execute_input":"2024-03-05T14:18:00.437270Z","iopub.status.idle":"2024-03-05T14:18:00.457802Z","shell.execute_reply.started":"2024-03-05T14:18:00.437219Z","shell.execute_reply":"2024-03-05T14:18:00.456298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from paddleocr import PaddleOCR,draw_ocr\nimport torch\nfrom PIL import Image\n\nocr = PaddleOCR(use_angle_cls=True, lang='en') \n# result = ocr.ocr(image_cv, cls=True)\n# print(result[0])\n\n\n# draw result\n# #image = Image.open(img_path).convert('RGB')\n# im_pil = Image.fromarray(img )\n# image = im_pil.convert('RGB')\n# boxes = [detection[0] for line in result for detection in line] # Nested loop added\n# txts = [detection[1][0] for line in result for detection in line] # Nested loop added\n# scores = [detection[1][1] for line in result for detection in line] # Nested loop added\n# # im_np = np.asarray(image)\n# im_show = draw_ocr(image, boxes, txts, scores,font_path = '/kaggle/input/fontsttf/doc_fonts_latin.ttf')\n# im_show = Image.fromarray(im_show)\n# display(im_show)\n# #image = Image.open(img_path).convert('RGB')\n\n# im_pil = Image.fromarray(rotate_image(img, -45) )\n# ocr = PaddleOCR(use_angle_cls=True, lang='en') # need to run only once to download and load model into memory\n# #img_path = '/kaggle/input/benetech-making-graphs-accessible/train/images/531cab0a3127.jpg'\n# result = ocr.ocr(rotate_image(image_cv, -45), cls=True)\n# for line in result:\n#     print(line)\n\n# image = im_pil.convert('RGB')\n# boxes = [detection[0] for line in result for detection in line] # Nested loop added\n# txts = [detection[1][0] for line in result for detection in line] # Nested loop added\n# scores = [detection[1][1] for line in result for detection in line] # Nested loop added\n# # im_np = np.asarray(image)\n# im_show = draw_ocr(image, boxes, txts, scores,font_path = '/kaggle/input/fontsttf/doc_fonts_latin.ttf')\n# im_show = Image.fromarray(im_show)\n# display(im_show)\n# # im_show.save('result.jpg')","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:18:00.460701Z","iopub.execute_input":"2024-03-05T14:18:00.461567Z","iopub.status.idle":"2024-03-05T14:18:49.899761Z","shell.execute_reply.started":"2024-03-05T14:18:00.461511Z","shell.execute_reply":"2024-03-05T14:18:49.897798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ocr = PaddleOCR(use_angle_cls=True, lang='en')\ndef generate_text(image_cv):\n    result = ocr.ocr(image_cv)[0]\n#     mat1 = invert_matrix(matrix_image(image_cv,45))\n#     img1 = rotate_image(image_cv,45)\n#     result1 = ocr.ocr(img1)[0]\n#     for i in range(len(result1)):\n#         for j in range(4):\n#             result1[i][0][j] = rotate_vector(result1[i][0][j],mat1)\n#     result.extend(result1)\n    mat2 = invert_matrix(matrix_image(image_cv,-45))\n    img2 = rotate_image(image_cv,-45)\n    result2 = ocr.ocr(img2)[0]\n    for i in range(len(result2)):\n        for j in range(len(result2[i][0])):\n            result2[i][0][j] = rotate_vector(result2[i][0][j],mat2)\n    result.extend(result2)\n    return [result]","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:24:30.804349Z","iopub.execute_input":"2024-03-05T14:24:30.804960Z","iopub.status.idle":"2024-03-05T14:24:30.815936Z","shell.execute_reply.started":"2024-03-05T14:24:30.804876Z","shell.execute_reply":"2024-03-05T14:24:30.814250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intersect_f1 = 0.5\nfrom shapely.geometry import Polygon\ndef get_f1(poly1,poly2):\n#     print(poly1,poly2)\n    p = Polygon(poly1).convex_hull\n    q = Polygon(poly2).convex_hull\n#     print(p,q)\n    intArea = p.intersection(q).area\n    return intArea*2/(p.area+q.area)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:57:05.144934Z","iopub.execute_input":"2024-03-05T14:57:05.146289Z","iopub.status.idle":"2024-03-05T14:57:05.153268Z","shell.execute_reply.started":"2024-03-05T14:57:05.146228Z","shell.execute_reply":"2024-03-05T14:57:05.152117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sift(words):\n    repeating = set()\n    for i in range(len(words)):\n        if not repeating.__contains__(i):\n            for j in range(i+1,len(words)):\n                if not repeating.__contains__(j):\n                    if(get_f1(words[i][0],words[j][0])>intersect_f1):\n                        if len(words[i][1][0])>len(words[j][1][0]):\n                            repeating.add(j)\n                        else:\n                            repeating.add(i)\n    answer = list()\n    for i in range(len(words)):\n        if not repeating.__contains__(i):\n            answer.append(words[i])\n    return answer","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:18:49.960365Z","iopub.execute_input":"2024-03-05T14:18:49.960777Z","iopub.status.idle":"2024-03-05T14:18:49.972655Z","shell.execute_reply.started":"2024-03-05T14:18:49.960747Z","shell.execute_reply":"2024-03-05T14:18:49.971249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_spell(words):\n    for i in range(len(words)):\n        numscnt = 0\n        for j in words[i][1][0]:\n            if '0123456789'.__contains__(j):# crap for only numbers\n                numscnt+=1\n        if numscnt/len(words[i][1][0])<0.5:\n            words[i][1] = (fix_spelling(words[i][1][0],max_length = max(len(words[i][1][0]),5)*3)[0]['generated_text'],words[i][1][1])\n            if words[i][1][0][-1]=='.':\n                words[i][1] = (words[i][1][0][:-1],words[i][1][1])\n\n    return words","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:18:49.974369Z","iopub.execute_input":"2024-03-05T14:18:49.974792Z","iopub.status.idle":"2024-03-05T14:18:49.986635Z","shell.execute_reply.started":"2024-03-05T14:18:49.974747Z","shell.execute_reply":"2024-03-05T14:18:49.985262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_image = \"/kaggle/input/benetech-making-graphs-accessible/train/images/662699d9da39.jpg\"\n# image_cv = cv2.imread(sample_image)\n# result = generate_text(image_cv)\n# # result1 = [sift(result0[0])]\n# # result = [fix_spell(result1[0])]\n# boxes = [detection[0] for line in result for detection in line] \n# txts = [detection[1][0] for line in result for detection in line]\n# scores = [detection[1][1] for line in result for detection in line]\n# # im_np = np.asarray(image)\n# print(result)\n# im_show = draw_ocr(image_cv, boxes, txts, scores,font_path = '/kaggle/input/fontsttf/doc_fonts_latin.ttf')\n# im_show = Image.fromarray(im_show)\n# display(im_show)\n    \n","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:56:52.656771Z","iopub.execute_input":"2024-03-05T14:56:52.657275Z","iopub.status.idle":"2024-03-05T14:56:52.664119Z","shell.execute_reply.started":"2024-03-05T14:56:52.657240Z","shell.execute_reply":"2024-03-05T14:56:52.662253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in result[0]:\n#     if len(i[0])!=4:\n#         print(i)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T14:29:02.097581Z","iopub.execute_input":"2024-03-05T14:29:02.098074Z","iopub.status.idle":"2024-03-05T14:29:02.105061Z","shell.execute_reply.started":"2024-03-05T14:29:02.098039Z","shell.execute_reply":"2024-03-05T14:29:02.103565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pipe(path):\n    image_cv = cv2.imread(path)\n    result0 = generate_text(image_cv)\n    result1 = [sift(result0[0])]\n    textsB = list()\n    for i in result1[0]:\n        textsB.append(i[1][0])\n    result = [fix_spell(result1[0])]\n    textsA = list()\n    for i in result[0]:\n        textsA.append(i[1][0])\n    boxes = [detection[0] for line in result for detection in line] \n    txts = [detection[1][0] for line in result for detection in line]\n    scores = [detection[1][1] for line in result for detection in line]\n    # im_np = np.asarray(image)\n    textsB = sorted(textsB)\n    textsA = sorted(textsA)\n    print(\"before spell checking:\"+'-'*200)\n    print(textsB)\n    print(\"after spell checking:\"+'-'*200)\n    print(textsA)\n    im_show = draw_ocr(image_cv, boxes, txts, scores,font_path = '/kaggle/input/fontsttf/doc_fonts_latin.ttf')\n    im_show = Image.fromarray(im_show)\n    #display(im_show)\npipe(sample_image)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T15:02:05.643440Z","iopub.execute_input":"2024-03-05T15:02:05.643959Z","iopub.status.idle":"2024-03-05T15:02:12.804381Z","shell.execute_reply.started":"2024-03-05T15:02:05.643922Z","shell.execute_reply":"2024-03-05T15:02:12.803012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt = 0\nfor i in os.scandir(path):\n    if cnt>=10:\n        break\n    cnt+=1\n    with open(\"/kaggle/input/benetech-making-graphs-accessible/train/annotations/\"+i.name[:-3]+\"json\", 'r') as f:\n        data = json.loads(f.read())\n        mdata = list()\n        #print(i.name)\n        #print(data)\n        for j in data['text']:\n            mdata.append(j['text'])\n        mdata= sorted(mdata)\n        pipe(path+'/'+i.name)\n        print(\"original:\"+'-'*200)\n        print(mdata,i.name)\n        print('*'*200)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T15:02:24.417501Z","iopub.execute_input":"2024-03-05T15:02:24.417985Z","iopub.status.idle":"2024-03-05T15:03:44.542266Z","shell.execute_reply.started":"2024-03-05T15:02:24.417949Z","shell.execute_reply":"2024-03-05T15:03:44.540471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}