fix(docreader): keep DOCX table content in parsed text (#2898)

* fix(docreader): keep DOCX table content in parsed text

Docx.__call__ returns tables separately from the text lines, and
parse_into_text used to drop them, silently losing all table content
in documents parsed with the builtin engine. Render the tables as GFM
Markdown (header row + delimiter + data rows) and append them to the
parsed text.

Fixes #2881

* fix(docreader): render DOCX tables from grid cells in document order

Skip the HTML round-trip that treated equal or empty adjacent cells as
colspan and dropped columns from GFM. Emit one column per python-docx
grid slot and interleave tables with paragraphs in body order.

---------

Co-authored-by: root <root@localhost.localdomain>
Co-authored-by: wizardchen <wizardchen@tencent.com>
main
LuckVd 2026-08-31 17:56:17 +08:00 committed by GitHub
parent 77245f0f4b
commit 81142dfd17
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 306 additions and 60 deletions

View File

@ -39,6 +39,7 @@ from docx.image.exceptions import (
UnexpectedEndOfFileError,
UnrecognizedImageError,
)
from docx.oxml.ns import qn
from PIL import Image
from docreader.config import CONFIG
@ -48,6 +49,41 @@ from docreader.utils import endecode
logger = logging.getLogger(__name__)
_W_P = qn("w:p")
_W_TBL = qn("w:tbl")
def _gfm_cell_text(text: Optional[str]) -> str:
"""Collapse whitespace and escape pipes so a cell stays one GFM column."""
return re.sub(r"\s+", " ", text or "").strip().replace("|", "\\|")
def table_to_gfm_markdown(table: Any) -> str:
"""Render a python-docx Table as GFM Markdown.
``row.cells`` already expands merged regions to one entry per grid slot
(repeating the same Cell object). Emitting every slot keeps column
alignment; comparing cell text would collapse adjacent equal/empty cells.
"""
rows: List[List[str]] = []
for row in table.rows:
cells = [_gfm_cell_text(cell.text) for cell in row.cells]
if cells:
rows.append(cells)
if not rows:
return ""
width = max(len(row) for row in rows)
if width == 0:
return ""
padded = [row + [""] * (width - len(row)) for row in rows]
lines = [
"| " + " | ".join(padded[0]) + " |",
"| " + " | ".join("---" for _ in padded[0]) + " |",
]
for row in padded[1:]:
lines.append("| " + " | ".join(row) + " |")
return "\n".join(lines)
class ImageData:
"""Represents a processed image of document content"""
@ -142,7 +178,7 @@ class DocxParser(BaseParser):
enable_multimodal=True,
upload_file=_inline_upload,
)
all_lines, tables = docx_processor(
all_lines, ordered_text = docx_processor(
binary=content,
max_workers=max_workers,
to_page=self.max_pages,
@ -150,7 +186,7 @@ class DocxParser(BaseParser):
processing_time = time.time() - start_time
logger.info(
f"Docx processing completed in {processing_time:.2f}s, "
f"extracted {len(all_lines)} sections and {len(tables)} tables"
f"extracted {len(all_lines)} sections"
)
logger.info("Processing document sections")
@ -187,7 +223,11 @@ class DocxParser(BaseParser):
f"Section processing completed in {section_processing_time:.2f}s"
)
logger.info("Combining all text parts")
text = "\n\n".join([part for part in text_parts if part])
# Prefer body-order markdown (paragraphs interleaved with tables).
# Fall back to page-section text if composition produced nothing.
text = (ordered_text or "").strip()
if not text:
text = "\n\n".join([part for part in text_parts if part])
# Check if the generated text is empty
if not text:
@ -481,7 +521,7 @@ class Docx:
from_page: int = 0,
to_page: int = 100000,
max_workers: Optional[int] = None,
) -> Tuple[List[LineData], List[Any]]:
) -> Tuple[List[LineData], str]:
"""
Process DOCX document, supporting concurrent processing of each page
@ -492,7 +532,8 @@ class Docx:
max_workers: Maximum number of workers, default to None (system decides)
Returns:
tuple: (List of LineData objects with document content, List of tables)
tuple: (List of LineData objects with document content,
body-order markdown with tables interleaved)
"""
logger.info("Processing DOCX document")
@ -503,7 +544,7 @@ class Docx:
# Load document
self.doc = self._load_document(binary)
if not self.doc:
return [], []
return [], ""
# Identify page structure
self.para_page_mapping = self._identify_page_paragraph_mapping(to_page)
@ -517,7 +558,7 @@ class Docx:
)
if not pages_to_process:
logger.warning("No pages to process after applying page limits!")
return [], []
return [], ""
# Initialize shared resources
self._init_shared_resources()
@ -531,17 +572,17 @@ class Docx:
max_workers,
)
# Process tables
tbls = self._process_tables()
ordered_text = self._compose_ordered_markdown(pages_to_process)
# Clean up document resources
self.doc = None
logger.info(
f"Document processing complete, "
f"extracted {len(self.all_lines)} text sections and {len(tbls)} tables"
f"extracted {len(self.all_lines)} text sections, "
f"{len(ordered_text)} characters of body-order markdown"
)
return self.all_lines, tbls
return self.all_lines, ordered_text
def _load_document(self, binary):
"""Load document
@ -1027,60 +1068,52 @@ class Docx:
except Exception as e:
logger.error(f"Failed to remove temporary file: {str(e)}")
def _process_tables(self):
"""Process tables in the document
def _compose_ordered_markdown(self, pages_to_process: List[int]) -> str:
"""Walk body paragraphs and tables in document order and emit GFM.
Returns:
list: List of tables
Paragraph processing is page-limited; tables are included only when they
sit within that same paragraph range so max_pages does not leak later
tables into the parsed text.
"""
tbls = []
table_count = len(self.doc.tables)
if table_count > 0:
logger.info(f"Processing {table_count} tables")
for tb_idx, tb in enumerate(self.doc.tables):
if tb_idx % 10 == 0: # Log only every 10 tables to reduce log volume
logger.info(f"Processing table {tb_idx + 1}/{table_count}")
allowed = set()
for page in pages_to_process:
allowed.update(self.para_page_mapping.get(page, []))
min_allowed = min(allowed) if allowed else 0
max_allowed = max(allowed) if allowed else -1
# Optimize: Check if table is empty
if len(tb.rows) == 0 or all(len(r.cells) == 0 for r in tb.rows):
logger.info(f"Skipping empty table {tb_idx + 1}")
continue
parts: List[str] = []
para_i = 0
tbl_i = 0
paragraphs = self.doc.paragraphs
tables = self.doc.tables
table_html = self._convert_table_to_html(tb)
# Still using tuple format for tables as they are handled differently
tbls.append(((None, table_html), ""))
for child in self.doc.element.body:
if child.tag == _W_P:
if para_i in allowed and para_i < len(paragraphs):
text = (paragraphs[para_i].text or "").strip()
if text:
parts.append(text)
para_i += 1
continue
if child.tag != _W_TBL:
continue
include_table = False
if not allowed:
include_table = True
elif min_allowed <= para_i <= max_allowed + 1:
include_table = True
if include_table and tbl_i < len(tables):
table = tables[tbl_i]
if not (
len(table.rows) == 0
or all(len(row.cells) == 0 for row in table.rows)
):
markdown = table_to_gfm_markdown(table)
if markdown:
parts.append(markdown)
tbl_i += 1
return tbls
def _convert_table_to_html(self, table):
"""Convert table to HTML
Args:
table: Table object
Returns:
str: HTML formatted table
"""
html = "<table>"
for r in table.rows:
html += "<tr>"
i = 0
while i < len(r.cells):
span = 1
c = r.cells[i]
for j in range(i + 1, len(r.cells)):
if c.text == r.cells[j].text:
span += 1
i = j
i += 1
html += (
f"<td>{c.text}</td>"
if span == 1
else f"<td colspan='{span}'>{c.text}</td>"
)
html += "</tr>"
html += "</table>"
return html
return "\n\n".join(parts)
def _safe_concat_images(self, images):
"""Safely concatenate image lists

View File

@ -0,0 +1,213 @@
import importlib.util
import io
import sys
import types
import unittest
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from unittest.mock import patch
from docx import Document as WordDocument
def _load_docx_parser():
"""Load docx_parser without triggering the heavy package __init__.
``docreader/parser/__init__.py`` imports doc_parser -> textract, a heavy
dependency unrelated to DOCX parsing. Registering the packages as bare
namespaces lets us import only the modules docx_parser actually needs.
"""
root = Path(__file__).resolve().parents[2]
docreader_pkg = types.ModuleType("docreader")
docreader_pkg.__path__ = [str(root / "docreader")]
sys.modules.setdefault("docreader", docreader_pkg)
parser_pkg = types.ModuleType("docreader.parser")
parser_pkg.__path__ = [str(root / "docreader" / "parser")]
sys.modules["docreader.parser"] = parser_pkg
spec = importlib.util.spec_from_file_location(
"docreader.parser.docx_parser", root / "docreader" / "parser" / "docx_parser.py"
)
module = importlib.util.module_from_spec(spec)
sys.modules["docreader.parser.docx_parser"] = module
spec.loader.exec_module(module)
return module
docx_parser = _load_docx_parser()
DocxParser = docx_parser.DocxParser
table_to_gfm_markdown = docx_parser.table_to_gfm_markdown
def _parse(content):
"""Parse a DOCX through the real Docx processor.
The production path uses a ProcessPoolExecutor backed by a multiprocessing
Manager. Some CI sandboxes forbid POSIX semaphores, so run the same page
task pool on threads and swap the manager for a plain list - parse
behavior is identical, only the execution backend changes.
"""
class _FakeManager:
def __enter__(self):
self._items = []
return self
def __exit__(self, *exc):
return False
def list(self):
return self._items
with patch.object(docx_parser, "Manager", _FakeManager), patch.object(
docx_parser, "ProcessPoolExecutor", ThreadPoolExecutor
):
return DocxParser(max_pages=100).parse_into_text(content)
def _docx_bytes(build):
doc = WordDocument()
build(doc)
buf = io.BytesIO()
doc.save(buf)
return buf.getvalue()
class DocxTableContentTest(unittest.TestCase):
"""Regression test: DOCX tables must be kept in the parsed text.
Docx.__call__ used to return tables separately from the text lines, and
parse_into_text dropped them, silently losing all table content.
"""
def _docx_with_table(self, cells):
def build(doc):
doc.add_paragraph("Introduction paragraph")
table = doc.add_table(rows=len(cells), cols=len(cells[0]))
for r, row in enumerate(cells):
for c, value in enumerate(row):
table.cell(r, c).text = value
return _docx_bytes(build)
def test_table_content_is_kept_in_parsed_text(self):
content = self._docx_with_table(
[["City", "Population"], ["Beijing", "21.5M"]]
)
document = _parse(content)
self.assertIn("Introduction paragraph", document.content)
for cell in ("City", "Population", "Beijing", "21.5M"):
self.assertIn(cell, document.content)
self.assertIn("| City | Population |", document.content)
self.assertIn("| --- | --- |", document.content)
def test_table_only_document_is_not_empty(self):
def build(doc):
table = doc.add_table(rows=2, cols=1)
table.cell(0, 0).text = "Header"
table.cell(1, 0).text = "Value"
document = _parse(_docx_bytes(build))
self.assertIn("Header", document.content)
self.assertIn("Value", document.content)
def test_pipe_in_cell_does_not_break_table(self):
def build(doc):
table = doc.add_table(rows=2, cols=1)
table.cell(0, 0).text = "A|B"
table.cell(1, 0).text = "C"
document = _parse(_docx_bytes(build))
self.assertIn(r"A\|B", document.content)
def test_empty_adjacent_cells_keep_columns(self):
def build(doc):
table = doc.add_table(rows=2, cols=3)
table.cell(0, 0).text = "A"
table.cell(0, 1).text = "B"
table.cell(0, 2).text = "C"
table.cell(1, 0).text = "1"
table.cell(1, 1).text = ""
table.cell(1, 2).text = ""
document = _parse(_docx_bytes(build))
self.assertIn("| A | B | C |", document.content)
self.assertIn("| 1 | | |", document.content)
def test_equal_adjacent_cells_are_not_collapsed(self):
"""Adjacent independent cells with the same text must stay separate.
Regression for the #2634 control row: ``相同值 | 相同值 | 独立值``.
"""
def build(doc):
table = doc.add_table(rows=2, cols=3)
table.cell(0, 0).text = "相同值"
table.cell(0, 1).text = "相同值"
table.cell(0, 2).text = "独立值"
table.cell(1, 0).text = "x"
table.cell(1, 1).text = "y"
table.cell(1, 2).text = "z"
document = _parse(_docx_bytes(build))
self.assertIn("| 相同值 | 相同值 | 独立值 |", document.content)
self.assertIn("| x | y | z |", document.content)
def test_horizontal_merge_keeps_grid_width(self):
def build(doc):
table = doc.add_table(rows=2, cols=3)
table.cell(0, 0).merge(table.cell(0, 1)).text = "merged"
table.cell(0, 2).text = "right"
table.cell(1, 0).text = "a"
table.cell(1, 1).text = "b"
table.cell(1, 2).text = "c"
document = _parse(_docx_bytes(build))
self.assertIn("| merged | merged | right |", document.content)
self.assertIn("| a | b | c |", document.content)
def test_table_stays_between_paragraphs(self):
def build(doc):
doc.add_paragraph("before table")
table = doc.add_table(rows=2, cols=1)
table.cell(0, 0).text = "H"
table.cell(1, 0).text = "V"
doc.add_paragraph("after table")
document = _parse(_docx_bytes(build))
before = document.content.index("before table")
header = document.content.index("| H |")
after = document.content.index("after table")
self.assertLess(before, header)
self.assertLess(header, after)
def test_html_like_cell_text_is_literal(self):
def build(doc):
table = doc.add_table(rows=1, cols=2)
table.cell(0, 0).text = "a < b"
table.cell(0, 1).text = "</td><td>injected"
document = _parse(_docx_bytes(build))
self.assertIn("| a < b | </td><td>injected |", document.content)
def test_table_to_gfm_markdown_helper(self):
doc = WordDocument()
table = doc.add_table(rows=2, cols=2)
table.cell(0, 0).text = "A"
table.cell(0, 1).text = "B"
table.cell(1, 0).text = "C"
table.cell(1, 1).text = "D"
self.assertEqual(
table_to_gfm_markdown(table),
"| A | B |\n| --- | --- |\n| C | D |",
)
def test_table_to_gfm_markdown_skips_empty(self):
class _Empty:
rows = []
self.assertEqual(table_to_gfm_markdown(_Empty()), "")
if __name__ == "__main__":
unittest.main()