From 81142dfd17b2778087e95d3a317483a2fd909b91 Mon Sep 17 00:00:00 2001 From: LuckVd <37114923+LuckVd@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:56:17 +0800 Subject: [PATCH] fix(docreader): keep DOCX table content in parsed text (#2898) * fix(docreader): keep DOCX table content in parsed text Docx.__call__ returns tables separately from the text lines, and parse_into_text used to drop them, silently losing all table content in documents parsed with the builtin engine. Render the tables as GFM Markdown (header row + delimiter + data rows) and append them to the parsed text. Fixes #2881 * fix(docreader): render DOCX tables from grid cells in document order Skip the HTML round-trip that treated equal or empty adjacent cells as colspan and dropped columns from GFM. Emit one column per python-docx grid slot and interleave tables with paragraphs in body order. --------- Co-authored-by: root Co-authored-by: wizardchen --- docreader/parser/docx_parser.py | 153 ++++++++++++-------- docreader/tests/test_docx_tables.py | 213 ++++++++++++++++++++++++++++ 2 files changed, 306 insertions(+), 60 deletions(-) create mode 100644 docreader/tests/test_docx_tables.py diff --git a/docreader/parser/docx_parser.py b/docreader/parser/docx_parser.py index bd713f80..f9820069 100644 --- a/docreader/parser/docx_parser.py +++ b/docreader/parser/docx_parser.py @@ -39,6 +39,7 @@ from docx.image.exceptions import ( UnexpectedEndOfFileError, UnrecognizedImageError, ) +from docx.oxml.ns import qn from PIL import Image from docreader.config import CONFIG @@ -48,6 +49,41 @@ from docreader.utils import endecode logger = logging.getLogger(__name__) +_W_P = qn("w:p") +_W_TBL = qn("w:tbl") + + +def _gfm_cell_text(text: Optional[str]) -> str: + """Collapse whitespace and escape pipes so a cell stays one GFM column.""" + return re.sub(r"\s+", " ", text or "").strip().replace("|", "\\|") + + +def table_to_gfm_markdown(table: Any) -> str: + """Render a python-docx Table as GFM Markdown. + + ``row.cells`` already expands merged regions to one entry per grid slot + (repeating the same Cell object). Emitting every slot keeps column + alignment; comparing cell text would collapse adjacent equal/empty cells. + """ + rows: List[List[str]] = [] + for row in table.rows: + cells = [_gfm_cell_text(cell.text) for cell in row.cells] + if cells: + rows.append(cells) + if not rows: + return "" + width = max(len(row) for row in rows) + if width == 0: + return "" + padded = [row + [""] * (width - len(row)) for row in rows] + lines = [ + "| " + " | ".join(padded[0]) + " |", + "| " + " | ".join("---" for _ in padded[0]) + " |", + ] + for row in padded[1:]: + lines.append("| " + " | ".join(row) + " |") + return "\n".join(lines) + class ImageData: """Represents a processed image of document content""" @@ -142,7 +178,7 @@ class DocxParser(BaseParser): enable_multimodal=True, upload_file=_inline_upload, ) - all_lines, tables = docx_processor( + all_lines, ordered_text = docx_processor( binary=content, max_workers=max_workers, to_page=self.max_pages, @@ -150,7 +186,7 @@ class DocxParser(BaseParser): processing_time = time.time() - start_time logger.info( f"Docx processing completed in {processing_time:.2f}s, " - f"extracted {len(all_lines)} sections and {len(tables)} tables" + f"extracted {len(all_lines)} sections" ) logger.info("Processing document sections") @@ -187,7 +223,11 @@ class DocxParser(BaseParser): f"Section processing completed in {section_processing_time:.2f}s" ) logger.info("Combining all text parts") - text = "\n\n".join([part for part in text_parts if part]) + # Prefer body-order markdown (paragraphs interleaved with tables). + # Fall back to page-section text if composition produced nothing. + text = (ordered_text or "").strip() + if not text: + text = "\n\n".join([part for part in text_parts if part]) # Check if the generated text is empty if not text: @@ -481,7 +521,7 @@ class Docx: from_page: int = 0, to_page: int = 100000, max_workers: Optional[int] = None, - ) -> Tuple[List[LineData], List[Any]]: + ) -> Tuple[List[LineData], str]: """ Process DOCX document, supporting concurrent processing of each page @@ -492,7 +532,8 @@ class Docx: max_workers: Maximum number of workers, default to None (system decides) Returns: - tuple: (List of LineData objects with document content, List of tables) + tuple: (List of LineData objects with document content, + body-order markdown with tables interleaved) """ logger.info("Processing DOCX document") @@ -503,7 +544,7 @@ class Docx: # Load document self.doc = self._load_document(binary) if not self.doc: - return [], [] + return [], "" # Identify page structure self.para_page_mapping = self._identify_page_paragraph_mapping(to_page) @@ -517,7 +558,7 @@ class Docx: ) if not pages_to_process: logger.warning("No pages to process after applying page limits!") - return [], [] + return [], "" # Initialize shared resources self._init_shared_resources() @@ -531,17 +572,17 @@ class Docx: max_workers, ) - # Process tables - tbls = self._process_tables() + ordered_text = self._compose_ordered_markdown(pages_to_process) # Clean up document resources self.doc = None logger.info( f"Document processing complete, " - f"extracted {len(self.all_lines)} text sections and {len(tbls)} tables" + f"extracted {len(self.all_lines)} text sections, " + f"{len(ordered_text)} characters of body-order markdown" ) - return self.all_lines, tbls + return self.all_lines, ordered_text def _load_document(self, binary): """Load document @@ -1027,60 +1068,52 @@ class Docx: except Exception as e: logger.error(f"Failed to remove temporary file: {str(e)}") - def _process_tables(self): - """Process tables in the document + def _compose_ordered_markdown(self, pages_to_process: List[int]) -> str: + """Walk body paragraphs and tables in document order and emit GFM. - Returns: - list: List of tables + Paragraph processing is page-limited; tables are included only when they + sit within that same paragraph range so max_pages does not leak later + tables into the parsed text. """ - tbls = [] - table_count = len(self.doc.tables) - if table_count > 0: - logger.info(f"Processing {table_count} tables") - for tb_idx, tb in enumerate(self.doc.tables): - if tb_idx % 10 == 0: # Log only every 10 tables to reduce log volume - logger.info(f"Processing table {tb_idx + 1}/{table_count}") + allowed = set() + for page in pages_to_process: + allowed.update(self.para_page_mapping.get(page, [])) + min_allowed = min(allowed) if allowed else 0 + max_allowed = max(allowed) if allowed else -1 - # Optimize: Check if table is empty - if len(tb.rows) == 0 or all(len(r.cells) == 0 for r in tb.rows): - logger.info(f"Skipping empty table {tb_idx + 1}") - continue + parts: List[str] = [] + para_i = 0 + tbl_i = 0 + paragraphs = self.doc.paragraphs + tables = self.doc.tables - table_html = self._convert_table_to_html(tb) - # Still using tuple format for tables as they are handled differently - tbls.append(((None, table_html), "")) + for child in self.doc.element.body: + if child.tag == _W_P: + if para_i in allowed and para_i < len(paragraphs): + text = (paragraphs[para_i].text or "").strip() + if text: + parts.append(text) + para_i += 1 + continue + if child.tag != _W_TBL: + continue + include_table = False + if not allowed: + include_table = True + elif min_allowed <= para_i <= max_allowed + 1: + include_table = True + if include_table and tbl_i < len(tables): + table = tables[tbl_i] + if not ( + len(table.rows) == 0 + or all(len(row.cells) == 0 for row in table.rows) + ): + markdown = table_to_gfm_markdown(table) + if markdown: + parts.append(markdown) + tbl_i += 1 - return tbls - - def _convert_table_to_html(self, table): - """Convert table to HTML - - Args: - table: Table object - - Returns: - str: HTML formatted table - """ - html = "" - for r in table.rows: - html += "" - i = 0 - while i < len(r.cells): - span = 1 - c = r.cells[i] - for j in range(i + 1, len(r.cells)): - if c.text == r.cells[j].text: - span += 1 - i = j - i += 1 - html += ( - f"" - if span == 1 - else f"" - ) - html += "" - html += "
{c.text}{c.text}
" - return html + return "\n\n".join(parts) def _safe_concat_images(self, images): """Safely concatenate image lists diff --git a/docreader/tests/test_docx_tables.py b/docreader/tests/test_docx_tables.py new file mode 100644 index 00000000..b3338d4f --- /dev/null +++ b/docreader/tests/test_docx_tables.py @@ -0,0 +1,213 @@ +import importlib.util +import io +import sys +import types +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +from docx import Document as WordDocument + + +def _load_docx_parser(): + """Load docx_parser without triggering the heavy package __init__. + + ``docreader/parser/__init__.py`` imports doc_parser -> textract, a heavy + dependency unrelated to DOCX parsing. Registering the packages as bare + namespaces lets us import only the modules docx_parser actually needs. + """ + root = Path(__file__).resolve().parents[2] + docreader_pkg = types.ModuleType("docreader") + docreader_pkg.__path__ = [str(root / "docreader")] + sys.modules.setdefault("docreader", docreader_pkg) + parser_pkg = types.ModuleType("docreader.parser") + parser_pkg.__path__ = [str(root / "docreader" / "parser")] + sys.modules["docreader.parser"] = parser_pkg + + spec = importlib.util.spec_from_file_location( + "docreader.parser.docx_parser", root / "docreader" / "parser" / "docx_parser.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules["docreader.parser.docx_parser"] = module + spec.loader.exec_module(module) + return module + + +docx_parser = _load_docx_parser() +DocxParser = docx_parser.DocxParser +table_to_gfm_markdown = docx_parser.table_to_gfm_markdown + + +def _parse(content): + """Parse a DOCX through the real Docx processor. + + The production path uses a ProcessPoolExecutor backed by a multiprocessing + Manager. Some CI sandboxes forbid POSIX semaphores, so run the same page + task pool on threads and swap the manager for a plain list - parse + behavior is identical, only the execution backend changes. + """ + + class _FakeManager: + def __enter__(self): + self._items = [] + return self + + def __exit__(self, *exc): + return False + + def list(self): + return self._items + + with patch.object(docx_parser, "Manager", _FakeManager), patch.object( + docx_parser, "ProcessPoolExecutor", ThreadPoolExecutor + ): + return DocxParser(max_pages=100).parse_into_text(content) + + +def _docx_bytes(build): + doc = WordDocument() + build(doc) + buf = io.BytesIO() + doc.save(buf) + return buf.getvalue() + + +class DocxTableContentTest(unittest.TestCase): + """Regression test: DOCX tables must be kept in the parsed text. + + Docx.__call__ used to return tables separately from the text lines, and + parse_into_text dropped them, silently losing all table content. + """ + + def _docx_with_table(self, cells): + def build(doc): + doc.add_paragraph("Introduction paragraph") + table = doc.add_table(rows=len(cells), cols=len(cells[0])) + for r, row in enumerate(cells): + for c, value in enumerate(row): + table.cell(r, c).text = value + + return _docx_bytes(build) + + def test_table_content_is_kept_in_parsed_text(self): + content = self._docx_with_table( + [["City", "Population"], ["Beijing", "21.5M"]] + ) + document = _parse(content) + self.assertIn("Introduction paragraph", document.content) + for cell in ("City", "Population", "Beijing", "21.5M"): + self.assertIn(cell, document.content) + self.assertIn("| City | Population |", document.content) + self.assertIn("| --- | --- |", document.content) + + def test_table_only_document_is_not_empty(self): + def build(doc): + table = doc.add_table(rows=2, cols=1) + table.cell(0, 0).text = "Header" + table.cell(1, 0).text = "Value" + + document = _parse(_docx_bytes(build)) + self.assertIn("Header", document.content) + self.assertIn("Value", document.content) + + def test_pipe_in_cell_does_not_break_table(self): + def build(doc): + table = doc.add_table(rows=2, cols=1) + table.cell(0, 0).text = "A|B" + table.cell(1, 0).text = "C" + + document = _parse(_docx_bytes(build)) + self.assertIn(r"A\|B", document.content) + + def test_empty_adjacent_cells_keep_columns(self): + def build(doc): + table = doc.add_table(rows=2, cols=3) + table.cell(0, 0).text = "A" + table.cell(0, 1).text = "B" + table.cell(0, 2).text = "C" + table.cell(1, 0).text = "1" + table.cell(1, 1).text = "" + table.cell(1, 2).text = "" + + document = _parse(_docx_bytes(build)) + self.assertIn("| A | B | C |", document.content) + self.assertIn("| 1 | | |", document.content) + + def test_equal_adjacent_cells_are_not_collapsed(self): + """Adjacent independent cells with the same text must stay separate. + + Regression for the #2634 control row: ``相同值 | 相同值 | 独立值``. + """ + + def build(doc): + table = doc.add_table(rows=2, cols=3) + table.cell(0, 0).text = "相同值" + table.cell(0, 1).text = "相同值" + table.cell(0, 2).text = "独立值" + table.cell(1, 0).text = "x" + table.cell(1, 1).text = "y" + table.cell(1, 2).text = "z" + + document = _parse(_docx_bytes(build)) + self.assertIn("| 相同值 | 相同值 | 独立值 |", document.content) + self.assertIn("| x | y | z |", document.content) + + def test_horizontal_merge_keeps_grid_width(self): + def build(doc): + table = doc.add_table(rows=2, cols=3) + table.cell(0, 0).merge(table.cell(0, 1)).text = "merged" + table.cell(0, 2).text = "right" + table.cell(1, 0).text = "a" + table.cell(1, 1).text = "b" + table.cell(1, 2).text = "c" + + document = _parse(_docx_bytes(build)) + self.assertIn("| merged | merged | right |", document.content) + self.assertIn("| a | b | c |", document.content) + + def test_table_stays_between_paragraphs(self): + def build(doc): + doc.add_paragraph("before table") + table = doc.add_table(rows=2, cols=1) + table.cell(0, 0).text = "H" + table.cell(1, 0).text = "V" + doc.add_paragraph("after table") + + document = _parse(_docx_bytes(build)) + before = document.content.index("before table") + header = document.content.index("| H |") + after = document.content.index("after table") + self.assertLess(before, header) + self.assertLess(header, after) + + def test_html_like_cell_text_is_literal(self): + def build(doc): + table = doc.add_table(rows=1, cols=2) + table.cell(0, 0).text = "a < b" + table.cell(0, 1).text = "injected" + + document = _parse(_docx_bytes(build)) + self.assertIn("| a < b | injected |", document.content) + + def test_table_to_gfm_markdown_helper(self): + doc = WordDocument() + table = doc.add_table(rows=2, cols=2) + table.cell(0, 0).text = "A" + table.cell(0, 1).text = "B" + table.cell(1, 0).text = "C" + table.cell(1, 1).text = "D" + self.assertEqual( + table_to_gfm_markdown(table), + "| A | B |\n| --- | --- |\n| C | D |", + ) + + def test_table_to_gfm_markdown_skips_empty(self): + class _Empty: + rows = [] + + self.assertEqual(table_to_gfm_markdown(_Empty()), "") + + +if __name__ == "__main__": + unittest.main()