fix(docreader): keep DOCX table content in parsed text (#2898)
* fix(docreader): keep DOCX table content in parsed text Docx.__call__ returns tables separately from the text lines, and parse_into_text used to drop them, silently losing all table content in documents parsed with the builtin engine. Render the tables as GFM Markdown (header row + delimiter + data rows) and append them to the parsed text. Fixes #2881 * fix(docreader): render DOCX tables from grid cells in document order Skip the HTML round-trip that treated equal or empty adjacent cells as colspan and dropped columns from GFM. Emit one column per python-docx grid slot and interleave tables with paragraphs in body order. --------- Co-authored-by: root <root@localhost.localdomain> Co-authored-by: wizardchen <wizardchen@tencent.com>main
parent
77245f0f4b
commit
81142dfd17
|
|
@ -39,6 +39,7 @@ from docx.image.exceptions import (
|
|||
UnexpectedEndOfFileError,
|
||||
UnrecognizedImageError,
|
||||
)
|
||||
from docx.oxml.ns import qn
|
||||
from PIL import Image
|
||||
|
||||
from docreader.config import CONFIG
|
||||
|
|
@ -48,6 +49,41 @@ from docreader.utils import endecode
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_W_P = qn("w:p")
|
||||
_W_TBL = qn("w:tbl")
|
||||
|
||||
|
||||
def _gfm_cell_text(text: Optional[str]) -> str:
|
||||
"""Collapse whitespace and escape pipes so a cell stays one GFM column."""
|
||||
return re.sub(r"\s+", " ", text or "").strip().replace("|", "\\|")
|
||||
|
||||
|
||||
def table_to_gfm_markdown(table: Any) -> str:
|
||||
"""Render a python-docx Table as GFM Markdown.
|
||||
|
||||
``row.cells`` already expands merged regions to one entry per grid slot
|
||||
(repeating the same Cell object). Emitting every slot keeps column
|
||||
alignment; comparing cell text would collapse adjacent equal/empty cells.
|
||||
"""
|
||||
rows: List[List[str]] = []
|
||||
for row in table.rows:
|
||||
cells = [_gfm_cell_text(cell.text) for cell in row.cells]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
if not rows:
|
||||
return ""
|
||||
width = max(len(row) for row in rows)
|
||||
if width == 0:
|
||||
return ""
|
||||
padded = [row + [""] * (width - len(row)) for row in rows]
|
||||
lines = [
|
||||
"| " + " | ".join(padded[0]) + " |",
|
||||
"| " + " | ".join("---" for _ in padded[0]) + " |",
|
||||
]
|
||||
for row in padded[1:]:
|
||||
lines.append("| " + " | ".join(row) + " |")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class ImageData:
|
||||
"""Represents a processed image of document content"""
|
||||
|
|
@ -142,7 +178,7 @@ class DocxParser(BaseParser):
|
|||
enable_multimodal=True,
|
||||
upload_file=_inline_upload,
|
||||
)
|
||||
all_lines, tables = docx_processor(
|
||||
all_lines, ordered_text = docx_processor(
|
||||
binary=content,
|
||||
max_workers=max_workers,
|
||||
to_page=self.max_pages,
|
||||
|
|
@ -150,7 +186,7 @@ class DocxParser(BaseParser):
|
|||
processing_time = time.time() - start_time
|
||||
logger.info(
|
||||
f"Docx processing completed in {processing_time:.2f}s, "
|
||||
f"extracted {len(all_lines)} sections and {len(tables)} tables"
|
||||
f"extracted {len(all_lines)} sections"
|
||||
)
|
||||
|
||||
logger.info("Processing document sections")
|
||||
|
|
@ -187,7 +223,11 @@ class DocxParser(BaseParser):
|
|||
f"Section processing completed in {section_processing_time:.2f}s"
|
||||
)
|
||||
logger.info("Combining all text parts")
|
||||
text = "\n\n".join([part for part in text_parts if part])
|
||||
# Prefer body-order markdown (paragraphs interleaved with tables).
|
||||
# Fall back to page-section text if composition produced nothing.
|
||||
text = (ordered_text or "").strip()
|
||||
if not text:
|
||||
text = "\n\n".join([part for part in text_parts if part])
|
||||
|
||||
# Check if the generated text is empty
|
||||
if not text:
|
||||
|
|
@ -481,7 +521,7 @@ class Docx:
|
|||
from_page: int = 0,
|
||||
to_page: int = 100000,
|
||||
max_workers: Optional[int] = None,
|
||||
) -> Tuple[List[LineData], List[Any]]:
|
||||
) -> Tuple[List[LineData], str]:
|
||||
"""
|
||||
Process DOCX document, supporting concurrent processing of each page
|
||||
|
||||
|
|
@ -492,7 +532,8 @@ class Docx:
|
|||
max_workers: Maximum number of workers, default to None (system decides)
|
||||
|
||||
Returns:
|
||||
tuple: (List of LineData objects with document content, List of tables)
|
||||
tuple: (List of LineData objects with document content,
|
||||
body-order markdown with tables interleaved)
|
||||
"""
|
||||
logger.info("Processing DOCX document")
|
||||
|
||||
|
|
@ -503,7 +544,7 @@ class Docx:
|
|||
# Load document
|
||||
self.doc = self._load_document(binary)
|
||||
if not self.doc:
|
||||
return [], []
|
||||
return [], ""
|
||||
|
||||
# Identify page structure
|
||||
self.para_page_mapping = self._identify_page_paragraph_mapping(to_page)
|
||||
|
|
@ -517,7 +558,7 @@ class Docx:
|
|||
)
|
||||
if not pages_to_process:
|
||||
logger.warning("No pages to process after applying page limits!")
|
||||
return [], []
|
||||
return [], ""
|
||||
|
||||
# Initialize shared resources
|
||||
self._init_shared_resources()
|
||||
|
|
@ -531,17 +572,17 @@ class Docx:
|
|||
max_workers,
|
||||
)
|
||||
|
||||
# Process tables
|
||||
tbls = self._process_tables()
|
||||
ordered_text = self._compose_ordered_markdown(pages_to_process)
|
||||
|
||||
# Clean up document resources
|
||||
self.doc = None
|
||||
|
||||
logger.info(
|
||||
f"Document processing complete, "
|
||||
f"extracted {len(self.all_lines)} text sections and {len(tbls)} tables"
|
||||
f"extracted {len(self.all_lines)} text sections, "
|
||||
f"{len(ordered_text)} characters of body-order markdown"
|
||||
)
|
||||
return self.all_lines, tbls
|
||||
return self.all_lines, ordered_text
|
||||
|
||||
def _load_document(self, binary):
|
||||
"""Load document
|
||||
|
|
@ -1027,60 +1068,52 @@ class Docx:
|
|||
except Exception as e:
|
||||
logger.error(f"Failed to remove temporary file: {str(e)}")
|
||||
|
||||
def _process_tables(self):
|
||||
"""Process tables in the document
|
||||
def _compose_ordered_markdown(self, pages_to_process: List[int]) -> str:
|
||||
"""Walk body paragraphs and tables in document order and emit GFM.
|
||||
|
||||
Returns:
|
||||
list: List of tables
|
||||
Paragraph processing is page-limited; tables are included only when they
|
||||
sit within that same paragraph range so max_pages does not leak later
|
||||
tables into the parsed text.
|
||||
"""
|
||||
tbls = []
|
||||
table_count = len(self.doc.tables)
|
||||
if table_count > 0:
|
||||
logger.info(f"Processing {table_count} tables")
|
||||
for tb_idx, tb in enumerate(self.doc.tables):
|
||||
if tb_idx % 10 == 0: # Log only every 10 tables to reduce log volume
|
||||
logger.info(f"Processing table {tb_idx + 1}/{table_count}")
|
||||
allowed = set()
|
||||
for page in pages_to_process:
|
||||
allowed.update(self.para_page_mapping.get(page, []))
|
||||
min_allowed = min(allowed) if allowed else 0
|
||||
max_allowed = max(allowed) if allowed else -1
|
||||
|
||||
# Optimize: Check if table is empty
|
||||
if len(tb.rows) == 0 or all(len(r.cells) == 0 for r in tb.rows):
|
||||
logger.info(f"Skipping empty table {tb_idx + 1}")
|
||||
continue
|
||||
parts: List[str] = []
|
||||
para_i = 0
|
||||
tbl_i = 0
|
||||
paragraphs = self.doc.paragraphs
|
||||
tables = self.doc.tables
|
||||
|
||||
table_html = self._convert_table_to_html(tb)
|
||||
# Still using tuple format for tables as they are handled differently
|
||||
tbls.append(((None, table_html), ""))
|
||||
for child in self.doc.element.body:
|
||||
if child.tag == _W_P:
|
||||
if para_i in allowed and para_i < len(paragraphs):
|
||||
text = (paragraphs[para_i].text or "").strip()
|
||||
if text:
|
||||
parts.append(text)
|
||||
para_i += 1
|
||||
continue
|
||||
if child.tag != _W_TBL:
|
||||
continue
|
||||
include_table = False
|
||||
if not allowed:
|
||||
include_table = True
|
||||
elif min_allowed <= para_i <= max_allowed + 1:
|
||||
include_table = True
|
||||
if include_table and tbl_i < len(tables):
|
||||
table = tables[tbl_i]
|
||||
if not (
|
||||
len(table.rows) == 0
|
||||
or all(len(row.cells) == 0 for row in table.rows)
|
||||
):
|
||||
markdown = table_to_gfm_markdown(table)
|
||||
if markdown:
|
||||
parts.append(markdown)
|
||||
tbl_i += 1
|
||||
|
||||
return tbls
|
||||
|
||||
def _convert_table_to_html(self, table):
|
||||
"""Convert table to HTML
|
||||
|
||||
Args:
|
||||
table: Table object
|
||||
|
||||
Returns:
|
||||
str: HTML formatted table
|
||||
"""
|
||||
html = "<table>"
|
||||
for r in table.rows:
|
||||
html += "<tr>"
|
||||
i = 0
|
||||
while i < len(r.cells):
|
||||
span = 1
|
||||
c = r.cells[i]
|
||||
for j in range(i + 1, len(r.cells)):
|
||||
if c.text == r.cells[j].text:
|
||||
span += 1
|
||||
i = j
|
||||
i += 1
|
||||
html += (
|
||||
f"<td>{c.text}</td>"
|
||||
if span == 1
|
||||
else f"<td colspan='{span}'>{c.text}</td>"
|
||||
)
|
||||
html += "</tr>"
|
||||
html += "</table>"
|
||||
return html
|
||||
return "\n\n".join(parts)
|
||||
|
||||
def _safe_concat_images(self, images):
|
||||
"""Safely concatenate image lists
|
||||
|
|
|
|||
|
|
@ -0,0 +1,213 @@
|
|||
import importlib.util
|
||||
import io
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from docx import Document as WordDocument
|
||||
|
||||
|
||||
def _load_docx_parser():
|
||||
"""Load docx_parser without triggering the heavy package __init__.
|
||||
|
||||
``docreader/parser/__init__.py`` imports doc_parser -> textract, a heavy
|
||||
dependency unrelated to DOCX parsing. Registering the packages as bare
|
||||
namespaces lets us import only the modules docx_parser actually needs.
|
||||
"""
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
docreader_pkg = types.ModuleType("docreader")
|
||||
docreader_pkg.__path__ = [str(root / "docreader")]
|
||||
sys.modules.setdefault("docreader", docreader_pkg)
|
||||
parser_pkg = types.ModuleType("docreader.parser")
|
||||
parser_pkg.__path__ = [str(root / "docreader" / "parser")]
|
||||
sys.modules["docreader.parser"] = parser_pkg
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"docreader.parser.docx_parser", root / "docreader" / "parser" / "docx_parser.py"
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules["docreader.parser.docx_parser"] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
docx_parser = _load_docx_parser()
|
||||
DocxParser = docx_parser.DocxParser
|
||||
table_to_gfm_markdown = docx_parser.table_to_gfm_markdown
|
||||
|
||||
|
||||
def _parse(content):
|
||||
"""Parse a DOCX through the real Docx processor.
|
||||
|
||||
The production path uses a ProcessPoolExecutor backed by a multiprocessing
|
||||
Manager. Some CI sandboxes forbid POSIX semaphores, so run the same page
|
||||
task pool on threads and swap the manager for a plain list - parse
|
||||
behavior is identical, only the execution backend changes.
|
||||
"""
|
||||
|
||||
class _FakeManager:
|
||||
def __enter__(self):
|
||||
self._items = []
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def list(self):
|
||||
return self._items
|
||||
|
||||
with patch.object(docx_parser, "Manager", _FakeManager), patch.object(
|
||||
docx_parser, "ProcessPoolExecutor", ThreadPoolExecutor
|
||||
):
|
||||
return DocxParser(max_pages=100).parse_into_text(content)
|
||||
|
||||
|
||||
def _docx_bytes(build):
|
||||
doc = WordDocument()
|
||||
build(doc)
|
||||
buf = io.BytesIO()
|
||||
doc.save(buf)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
class DocxTableContentTest(unittest.TestCase):
|
||||
"""Regression test: DOCX tables must be kept in the parsed text.
|
||||
|
||||
Docx.__call__ used to return tables separately from the text lines, and
|
||||
parse_into_text dropped them, silently losing all table content.
|
||||
"""
|
||||
|
||||
def _docx_with_table(self, cells):
|
||||
def build(doc):
|
||||
doc.add_paragraph("Introduction paragraph")
|
||||
table = doc.add_table(rows=len(cells), cols=len(cells[0]))
|
||||
for r, row in enumerate(cells):
|
||||
for c, value in enumerate(row):
|
||||
table.cell(r, c).text = value
|
||||
|
||||
return _docx_bytes(build)
|
||||
|
||||
def test_table_content_is_kept_in_parsed_text(self):
|
||||
content = self._docx_with_table(
|
||||
[["City", "Population"], ["Beijing", "21.5M"]]
|
||||
)
|
||||
document = _parse(content)
|
||||
self.assertIn("Introduction paragraph", document.content)
|
||||
for cell in ("City", "Population", "Beijing", "21.5M"):
|
||||
self.assertIn(cell, document.content)
|
||||
self.assertIn("| City | Population |", document.content)
|
||||
self.assertIn("| --- | --- |", document.content)
|
||||
|
||||
def test_table_only_document_is_not_empty(self):
|
||||
def build(doc):
|
||||
table = doc.add_table(rows=2, cols=1)
|
||||
table.cell(0, 0).text = "Header"
|
||||
table.cell(1, 0).text = "Value"
|
||||
|
||||
document = _parse(_docx_bytes(build))
|
||||
self.assertIn("Header", document.content)
|
||||
self.assertIn("Value", document.content)
|
||||
|
||||
def test_pipe_in_cell_does_not_break_table(self):
|
||||
def build(doc):
|
||||
table = doc.add_table(rows=2, cols=1)
|
||||
table.cell(0, 0).text = "A|B"
|
||||
table.cell(1, 0).text = "C"
|
||||
|
||||
document = _parse(_docx_bytes(build))
|
||||
self.assertIn(r"A\|B", document.content)
|
||||
|
||||
def test_empty_adjacent_cells_keep_columns(self):
|
||||
def build(doc):
|
||||
table = doc.add_table(rows=2, cols=3)
|
||||
table.cell(0, 0).text = "A"
|
||||
table.cell(0, 1).text = "B"
|
||||
table.cell(0, 2).text = "C"
|
||||
table.cell(1, 0).text = "1"
|
||||
table.cell(1, 1).text = ""
|
||||
table.cell(1, 2).text = ""
|
||||
|
||||
document = _parse(_docx_bytes(build))
|
||||
self.assertIn("| A | B | C |", document.content)
|
||||
self.assertIn("| 1 | | |", document.content)
|
||||
|
||||
def test_equal_adjacent_cells_are_not_collapsed(self):
|
||||
"""Adjacent independent cells with the same text must stay separate.
|
||||
|
||||
Regression for the #2634 control row: ``相同值 | 相同值 | 独立值``.
|
||||
"""
|
||||
|
||||
def build(doc):
|
||||
table = doc.add_table(rows=2, cols=3)
|
||||
table.cell(0, 0).text = "相同值"
|
||||
table.cell(0, 1).text = "相同值"
|
||||
table.cell(0, 2).text = "独立值"
|
||||
table.cell(1, 0).text = "x"
|
||||
table.cell(1, 1).text = "y"
|
||||
table.cell(1, 2).text = "z"
|
||||
|
||||
document = _parse(_docx_bytes(build))
|
||||
self.assertIn("| 相同值 | 相同值 | 独立值 |", document.content)
|
||||
self.assertIn("| x | y | z |", document.content)
|
||||
|
||||
def test_horizontal_merge_keeps_grid_width(self):
|
||||
def build(doc):
|
||||
table = doc.add_table(rows=2, cols=3)
|
||||
table.cell(0, 0).merge(table.cell(0, 1)).text = "merged"
|
||||
table.cell(0, 2).text = "right"
|
||||
table.cell(1, 0).text = "a"
|
||||
table.cell(1, 1).text = "b"
|
||||
table.cell(1, 2).text = "c"
|
||||
|
||||
document = _parse(_docx_bytes(build))
|
||||
self.assertIn("| merged | merged | right |", document.content)
|
||||
self.assertIn("| a | b | c |", document.content)
|
||||
|
||||
def test_table_stays_between_paragraphs(self):
|
||||
def build(doc):
|
||||
doc.add_paragraph("before table")
|
||||
table = doc.add_table(rows=2, cols=1)
|
||||
table.cell(0, 0).text = "H"
|
||||
table.cell(1, 0).text = "V"
|
||||
doc.add_paragraph("after table")
|
||||
|
||||
document = _parse(_docx_bytes(build))
|
||||
before = document.content.index("before table")
|
||||
header = document.content.index("| H |")
|
||||
after = document.content.index("after table")
|
||||
self.assertLess(before, header)
|
||||
self.assertLess(header, after)
|
||||
|
||||
def test_html_like_cell_text_is_literal(self):
|
||||
def build(doc):
|
||||
table = doc.add_table(rows=1, cols=2)
|
||||
table.cell(0, 0).text = "a < b"
|
||||
table.cell(0, 1).text = "</td><td>injected"
|
||||
|
||||
document = _parse(_docx_bytes(build))
|
||||
self.assertIn("| a < b | </td><td>injected |", document.content)
|
||||
|
||||
def test_table_to_gfm_markdown_helper(self):
|
||||
doc = WordDocument()
|
||||
table = doc.add_table(rows=2, cols=2)
|
||||
table.cell(0, 0).text = "A"
|
||||
table.cell(0, 1).text = "B"
|
||||
table.cell(1, 0).text = "C"
|
||||
table.cell(1, 1).text = "D"
|
||||
self.assertEqual(
|
||||
table_to_gfm_markdown(table),
|
||||
"| A | B |\n| --- | --- |\n| C | D |",
|
||||
)
|
||||
|
||||
def test_table_to_gfm_markdown_skips_empty(self):
|
||||
class _Empty:
|
||||
rows = []
|
||||
|
||||
self.assertEqual(table_to_gfm_markdown(_Empty()), "")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue