{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":14523528,"datasetId":9275882,"databundleVersionId":15351527},{"sourceType":"datasetVersion","sourceId":14523367,"datasetId":9275788,"databundleVersionId":15351355}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Download and install the necessary packages","metadata":{}},{"cell_type":"code","source":"!pip install pymupdf\n!pip install numba\n\n!apt-get install poppler-utils\n!pip install pytesseract opencv-python pdf2image pillow numpy\n!pip install pylatexenc","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Code","metadata":{}},{"cell_type":"code","source":"# Imports\nimport io\nimport shutil\nimport os\n\nimport cv2\n\nfrom PIL import Image, ImageDraw, ImageFont\nfrom pdf2image import convert_from_path\nimport numpy as np\nimport pytesseract\nfrom pytesseract import Output\nimport pymupdf\nimport matplotlib.pyplot as plt\n\n# ---------------------------------------------------------------------------\n# Core Functions\n# ---------------------------------------------------------------------------\n\ndef get_pdf_page_image(pdf_path, page_number, dpi=300):\n    \"\"\"Extracts a page from a PDF as a PIL Image.\"\"\"\n    try:\n        doc = pymupdf.open(pdf_path)\n        page = doc.load_page(page_number - 1)\n        pix = page.get_pixmap(dpi=dpi)\n        img_data = pix.tobytes(\"png\")\n        return Image.open(io.BytesIO(img_data))\n    except Exception as e:\n        print(f\"PyMuPDF failed, falling back to pdf2image: {e}\")\n        try:\n            images = convert_from_path(pdf_path, first_page=page_number, last_page=page_number, dpi=dpi)\n            return images[0]\n        except Exception as e2:\n            print(f\"pdf2image failed: {e2}\")\n            return None\n\ndef filter_and_pair_table_rows(refined_fields, w_img):\n    \"\"\"\n    Filters out headers/comments, groups fields into table rows, \n    and pairs the Akkadian (Left) with the English (Right) text.\n    \"\"\"\n    max_col_width = 0.6 * w_img\n    narrow_fields = [f for f in refined_fields if f[2] <= max_col_width]\n    \n    rows = []\n    for f in narrow_fields:\n        x, y, w, h = f\n        placed = False\n        for row in rows:\n            row_y = row['y']\n            row_h = row['h']\n            overlap = max(0, min(y + h, row_y + row_h) - max(y, row_y))\n            if overlap > 0.4 * min(h, row_h):\n                row['fields'].append(f)\n                row['y'] = min(row['y'], y)\n                row['h'] = max(row_y + row_h, y + h) - row['y']\n                placed = True\n                break\n        if not placed:\n            rows.append({'y': y, 'h': h, 'fields': [f]})\n\n    table_y_min = float('inf')\n    table_y_max = 0\n    \n    for row in rows:\n        has_left, has_right = False, False\n        for (x, y, w, h) in row['fields']:\n            cx = x + w / 2\n            if cx < 0.45 * w_img:\n                has_left = True\n            elif cx > 0.55 * w_img:\n                has_right = True\n        \n        if has_left and has_right:\n            table_y_min = min(table_y_min, row['y'])\n            table_y_max = max(table_y_max, row['y'] + row['h'])\n\n    margin = 30\n    table_rows_raw = [r for r in rows if r['y'] >= (table_y_min - margin) and (r['y'] + r['h']) <= (table_y_max + margin)]\n    table_rows_raw.sort(key=lambda r: r['y'])\n\n    structured_rows = []\n    for row_idx, row in enumerate(table_rows_raw):\n        left_fields, right_fields = [], []\n        \n        for f in row['fields']:\n            cx = f[0] + f[2] / 2\n            if cx < 0.45 * w_img:\n                left_fields.append(f)\n            elif cx > 0.55 * w_img:\n                right_fields.append(f)\n                \n        def merge_fields(flist, max_gap=0.01 * w_img):\n            if not flist: return None\n            \n            flist = sorted(flist, key=lambda f: f[0])\n            \n            clusters = []\n            current_cluster = [flist[0]]\n            x_end = flist[0][0] + flist[0][2]\n            \n            for f in flist[1:]:\n                f_x, f_y, f_w, f_h = f\n                gap = f_x - x_end\n                \n                if gap > max_gap:\n                    clusters.append(current_cluster)\n                    current_cluster = [f]\n                else:\n                    current_cluster.append(f)\n                \n                x_end = max(x_end, f_x + f_w)\n                \n            clusters.append(current_cluster)\n            \n            valid_clusters = []\n            for i, c in enumerate(clusters):\n                c_w = max(f[0] + f[2] for f in c) - min(f[0] for f in c)\n                is_edge = (i == 0) or (i == len(clusters) - 1)\n                is_small = c_w < 0.06 * w_img\n                \n                if is_edge and is_small and len(c) <= 2 and len(clusters) > 1:\n                    continue\n                valid_clusters.append(c)\n            \n            if not valid_clusters:\n                valid_clusters = [max(clusters, key=lambda c: max(f[0] + f[2] for f in c) - min(f[0] for f in c))]\n                \n            x1 = min(f[0] for c in valid_clusters for f in c)\n            y1 = min(f[1] for c in valid_clusters for f in c)\n            x2 = max(f[0] + f[2] for c in valid_clusters for f in c)\n            y2 = max(f[1] + f[3] for c in valid_clusters for f in c)\n                \n            return (x1, y1, x2 - x1, y2 - y1)\n            \n        left_merged = merge_fields(left_fields)\n        right_merged = merge_fields(right_fields)\n        \n        if left_merged or right_merged:\n            structured_rows.append({\n                'row_idx': row_idx,\n                'left': left_merged,\n                'right': right_merged\n            })\n            \n    return structured_rows\n\ndef extract_fields_mod(pdf_path, page_num, output_dir, psm=3, bottom_trim=0.0, pdf_idx=0, show_bounds=False, show_plot=False):\n    \"\"\"Performs layout analysis using Tesseract bounding boxes and refines the fields.\"\"\"\n    if not os.path.exists(output_dir):\n        os.makedirs(output_dir, exist_ok=True)\n\n    pil_img = get_pdf_page_image(pdf_path, page_num)\n    if pil_img is None:\n        print(f\"Error: Could not load image for page {page_num}.\")\n        return None\n\n    img = np.array(pil_img)\n    if len(img.shape) == 2:\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)\n    else:\n        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n\n    # Apply dynamic bottom crop (e.g., bottom_trim=0.10 means keep top 90%)\n    crop_h = int(img.shape[0] * (1.0 - bottom_trim))\n    img = img[:crop_h, :]\n\n    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    _, bin_inv = cv2.threshold(img_gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)\n    h_img, w_img = bin_inv.shape\n\n    img_thresh = cv2.adaptiveThreshold(img_gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 11, 2)\n\n    custom_config = r'' + f'--psm {psm}'\n    try:\n        d = pytesseract.image_to_data(img_thresh, config=custom_config, output_type=Output.DICT)\n    except Exception as e:\n        print(f\"Tesseract Error on page {page_num}: {e}\")\n        return []\n\n    lines_map = {}\n    n_boxes = len(d['level'])\n\n    for i in range(n_boxes):\n        if d['level'][i] != 5:\n            continue\n        text = d['text'][i].strip()\n        if not text:\n            continue\n        key = (d['block_num'][i], d['par_num'][i], d['line_num'][i])\n        if key not in lines_map:\n            lines_map[key] = []\n        lines_map[key].append({'left': d['left'][i], 'top': d['top'][i], 'width': d['width'][i], 'height': d['height'][i]})\n\n    raw_fields = []\n    for key, boxes in lines_map.items():\n        x1 = min(b['left'] for b in boxes)\n        y1 = min(b['top'] for b in boxes)\n        x2 = max(b['left'] + b['width'] for b in boxes)\n        y2 = max(b['top'] + b['height'] for b in boxes)\n        raw_fields.append((x1, y1, x2 - x1, y2 - y1))\n\n    # Refine Fields\n    refined_fields = []\n    for (x, y, w, h) in raw_fields:\n        mid_y = min(max(y + h // 2, 0), h_img - 1)\n        mid_row = bin_inv[mid_y, x:x+w]\n        if len(mid_row) == 0:\n            refined_fields.append((x, y, w, h))\n            continue\n            \n        max_px_val = np.max(mid_row)\n        if max_px_val == 0:\n            refined_fields.append((x, y, w, h))\n            continue\n\n        cur_x, cur_y, cur_w, cur_h = x, y, w, h\n\n        while cur_h > 1 and not np.any(bin_inv[cur_y, cur_x:cur_x+cur_w] == max_px_val): cur_y, cur_h = cur_y + 1, cur_h - 1\n        while cur_h > 1 and not np.any(bin_inv[cur_y+cur_h-1, cur_x:cur_x+cur_w] == max_px_val): cur_h -= 1\n        while cur_w > 1 and not np.any(bin_inv[cur_y:cur_y+cur_h, cur_x] == max_px_val): cur_x, cur_w = cur_x + 1, cur_w - 1\n        while cur_w > 1 and not np.any(bin_inv[cur_y:cur_y+cur_h, cur_x+cur_w-1] == max_px_val): cur_w -= 1\n\n        for _ in range(0, 10, 2):\n            if cur_y <= 0 or not np.any(bin_inv[cur_y, cur_x:cur_x+cur_w] == max_px_val): break\n            s = min(2, cur_y); cur_y, cur_h = cur_y - s, cur_h + s\n        for _ in range(0, 10, 2):\n            if cur_y+cur_h >= h_img or not np.any(bin_inv[cur_y+cur_h-1, cur_x:cur_x+cur_w] == max_px_val): break\n            s = min(2, h_img - (cur_y+cur_h)); cur_h += s\n        for _ in range(0, 10, 2):\n            if cur_x <= 0 or not np.any(bin_inv[cur_y:cur_y+cur_h, cur_x] == max_px_val): break\n            s = min(2, cur_x); cur_x, cur_w = cur_x - s, cur_w + s\n        for _ in range(0, 10, 2):\n            if cur_x+cur_w >= w_img or not np.any(bin_inv[cur_y:cur_y+cur_h, cur_x+cur_w-1] == max_px_val): break\n            s = min(2, w_img - (cur_x+cur_w)); cur_w += s\n\n        refined_fields.append((cur_x, cur_y, cur_w, cur_h))\n\n    # Table Filtering\n    structured_table = filter_and_pair_table_rows(refined_fields, w_img)\n    if not structured_table:\n        return []\n\n    vis_img = img.copy()\n    \n    # Save individual crops with pdf_idx identifier to prevent collisions\n    for row in structured_table:\n        r_idx = row['row_idx']\n        if row['left']:\n            x, y, w, h = row['left']\n            cv2.imwrite(os.path.join(output_dir, f\"pdf-{pdf_idx}_p-{page_num}_r-{r_idx}_left.png\"), img[y:y+h, x:x+w])\n            if show_bounds:\n                cv2.rectangle(vis_img, (x, y), (x + w, y + h), (0, 0, 255), 2)\n        if row['right']:\n            x, y, w, h = row['right']\n            cv2.imwrite(os.path.join(output_dir, f\"pdf-{pdf_idx}_p-{page_num}_r-{r_idx}_right.png\"), img[y:y+h, x:x+w])\n            if show_bounds:\n                cv2.rectangle(vis_img, (x, y), (x + w, y + h), (255, 0, 0), 2)\n\n    if show_plot:\n        plt.figure(figsize=(12, 16))\n        plt.imshow(cv2.cvtColor(vis_img, cv2.COLOR_BGR2RGB))\n        plt.axis('off')\n        plt.title(f\"PDF {pdf_idx} Page {page_num} Fields\")\n        plt.show()\n\n    return structured_table\n\ndef build_table_pdf_page(page_num, structured_table, img_dir, pdf_idx, pdf_name):\n    \"\"\"Generates a PIL Image visualizing the side-by-side table for a single PDF page.\"\"\"\n    margin = 50\n    padding = 20\n    col_width = 1200\n    page_width = (margin * 2) + (col_width * 2)\n    header_height = 200\n    \n    total_height = margin + header_height\n    row_images = []\n    \n    for row in structured_table:\n        r_idx = row['row_idx']\n        left_path = os.path.join(img_dir, f\"pdf-{pdf_idx}_p-{page_num}_r-{r_idx}_left.png\")\n        right_path = os.path.join(img_dir, f\"pdf-{pdf_idx}_p-{page_num}_r-{r_idx}_right.png\")\n        \n        left_img = Image.open(left_path) if os.path.exists(left_path) else None\n        right_img = Image.open(right_path) if os.path.exists(right_path) else None\n        \n        lh = left_img.height if left_img else 0\n        rh = right_img.height if right_img else 0\n        \n        row_height = max(lh, rh) + (padding * 2)\n        row_images.append({'left': left_img, 'right': right_img, 'height': row_height})\n        total_height += row_height\n        \n    total_height += margin\n    \n    canvas = Image.new('RGB', (page_width, total_height), 'white')\n    draw = ImageDraw.Draw(canvas)\n    \n    try:\n        font_large = ImageFont.truetype(\"arial.ttf\", 40)\n    except IOError:\n        font_large = ImageFont.load_default()\n\n    # Draw Header (Now includes PDF name)\n    header_text = f\"TABLE [{pdf_name} | PAGE NO. {page_num}]\"\n    draw.text((margin, margin), header_text, fill=\"black\", font=font_large)\n    \n    current_y = margin + header_height\n    \n    draw.rectangle([margin, current_y, margin + (col_width * 2), current_y + sum(r['height'] for r in row_images)], outline=\"black\", width=3)\n    draw.line([(margin + col_width, current_y), (margin + col_width, current_y + sum(r['height'] for r in row_images))], fill=\"black\", width=3)\n\n    for row_data in row_images:\n        if row_data['left']:\n            img = row_data['left']\n            paste_y = current_y + padding + ((row_data['height'] - (padding*2)) - img.height) // 2\n            canvas.paste(img, (margin + padding, paste_y))\n            \n        if row_data['right']:\n            img = row_data['right']\n            paste_y = current_y + padding + ((row_data['height'] - (padding*2)) - img.height) // 2\n            canvas.paste(img, (margin + col_width + padding, paste_y))\n            \n        current_y += row_data['height']\n        draw.line([(margin, current_y), (margin + (col_width * 2), current_y)], fill=\"black\", width=2)\n        \n    return canvas\n\ndef parse_page_range(range_str, max_pages):\n    \"\"\"Parses a string like '43-57,59-82' into a sorted list of unique page integers.\"\"\"\n    if not range_str or not range_str.strip():\n        return list(range(1, max_pages + 1))\n        \n    pages = set()\n    for part in range_str.split(','):\n        part = part.strip()\n        if not part: continue\n        if '-' in part:\n            start, end = part.split('-', 1)\n            start = max(1, int(start.strip()))\n            end = min(max_pages, int(end.strip()))\n            pages.update(range(start, end + 1))\n        else:\n            p = int(part.strip())\n            if 1 <= p <= max_pages:\n                pages.add(p)\n    return sorted(list(pages))\n\n# ---------------------------------------------------------------------------\n# Execution Block\n# ---------------------------------------------------------------------------\n\ndef main():\n    # 1. Configuration: Add as many PDFs as you want in this list\n    PDF_CONFIGS = [\n        {\n            \"pdf_path\": '/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 5 2008.pdf',\n            \"page_range\": \"43-168\", \n            \"bottom_trim\": 0.10 #REMOVE FOOTNOTES\n        },# 6 SERIES ALREADY DON'T HAVE FOOT NOTES -> SO, NO BOTTOM TRIM\n        {\n            \"pdf_path\": '/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 6a.pdf',\n            \"page_range\": \"52-436\", \n            \"bottom_trim\": 0.00\n        },\n        {\n            \"pdf_path\": '/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 6b.pdf',\n            \"page_range\": \"59-352\", \n            \"bottom_trim\": 0.00\n        },\n        {\n            \"pdf_path\": '/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 6c.pdf',\n            \"page_range\": \"46-297\", \n            \"bottom_trim\": 0.00\n        },\n        {\n            \"pdf_path\": '/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 6d.pdf',\n            \"page_range\": \"26-147\", \n            \"bottom_trim\": 0.00\n        },\n        {\n            \"pdf_path\": '/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 6e.pdf',\n            \"page_range\": \"99-266\", \n            \"bottom_trim\": 0.00\n        },\n        {\n            \"pdf_path\": '/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT 8 2015.pdf',\n            \"page_range\": \"49-517\", \n            \"bottom_trim\": 0.10\n        },\n        {\n            \"pdf_path\": '/kaggle/input/datasets/deeppast/old-assyrian-kltepe-tablets-in-pdf/AKT_12.pdf',\n            \"page_range\": \"43-174\", \n            \"bottom_trim\": 0.10\n        },\n    ]\n\n    RAW_EXTRACTED_LINES = \"/kaggle/working/extracted_fields_layout\"\n    OUTPUT_PDF_DIR = \"/kaggle/working/Processed_PDFs\"\n    MAX_PAGES_PER_PDF = 50\n    \n    os.makedirs(RAW_EXTRACTED_LINES, exist_ok=True)\n    os.makedirs(OUTPUT_PDF_DIR, exist_ok=True)\n    \n    print(\"Initializing Sequential PDF Processing...\")\n    \n    for pdf_idx, config in enumerate(PDF_CONFIGS):\n        pdf_path = config[\"pdf_path\"]\n        page_range_str = config[\"page_range\"]\n        bottom_trim = config[\"bottom_trim\"]\n        \n        if not os.path.exists(pdf_path):\n            print(f\"Skipping {pdf_path}: File not found.\")\n            continue\n            \n        pdf_name = os.path.basename(pdf_path)\n        doc = pymupdf.open(pdf_path)\n        max_pages = len(doc)\n        doc.close()\n        \n        pages_to_process = parse_page_range(page_range_str, max_pages)\n        print(f\"\\nQueued {len(pages_to_process)} pages from {pdf_name}...\")\n        \n        compiled_page_paths = []\n        \n        for page_num in pages_to_process:\n            print(f\"[*] Processing: {pdf_name} -> Page {page_num}\")\n            \n            structured_table = extract_fields_mod(\n                pdf_path=pdf_path, \n                page_num=page_num, \n                output_dir=RAW_EXTRACTED_LINES, \n                psm=11, \n                bottom_trim=bottom_trim,\n                pdf_idx=pdf_idx,\n                show_bounds=False,\n                show_plot=False\n            )\n            \n            if not structured_table:\n                continue\n                \n            # Build visual PDF image for this page\n            page_img = build_table_pdf_page(page_num, structured_table, RAW_EXTRACTED_LINES, pdf_idx, pdf_name)\n            \n            # Save the compiled page image to disk\n            out_path = os.path.join(RAW_EXTRACTED_LINES, f\"compiled_pdf_{pdf_idx}_p_{page_num}.png\")\n            page_img.save(out_path)\n            compiled_page_paths.append(out_path)\n            \n        # Stitch Output Together into final PDFs for THIS document\n        if not compiled_page_paths:\n            print(f\"No tables successfully extracted for: {pdf_name}\")\n            continue\n            \n        base_name = os.path.splitext(pdf_name)[0]\n        \n        # Chunk the compiled pages into lists of max MAX_PAGES_PER_PDF\n        chunks = [compiled_page_paths[i:i + MAX_PAGES_PER_PDF] for i in range(0, len(compiled_page_paths), MAX_PAGES_PER_PDF)]\n        \n        for part_idx, chunk in enumerate(chunks):\n            if len(chunks) == 1:\n                out_filename = f\"{base_name}_Processed.pdf\"\n            else:\n                out_filename = f\"{base_name}_Processed_Part-{part_idx + 1}.pdf\"\n                \n            out_filepath = os.path.join(OUTPUT_PDF_DIR, out_filename)\n            \n            # Open the first image\n            first_page = Image.open(chunk[0]).convert('RGB')\n            \n            # Load subsequent images as a list\n            append_images = [Image.open(p).convert('RGB') for p in chunk[1:]]\n                \n            # Save as single PDF\n            first_page.save(\n                out_filepath, \n                \"PDF\", \n                resolution=100.0, \n                save_all=True,\n                append_images=append_images\n            )\n            print(f\"Saved {len(chunk)} pages to {out_filepath}\")\n\n    print(\"\\nSequential processing complete!\")\n\nif __name__ == '__main__':\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-14T18:36:53.280898Z","iopub.execute_input":"2026-03-14T18:36:53.281267Z","iopub.status.idle":"2026-03-14T18:37:51.705566Z","shell.execute_reply.started":"2026-03-14T18:36:53.281237Z","shell.execute_reply":"2026-03-14T18:37:51.704514Z"}},"outputs":[],"execution_count":null}]}