From 578ca1b5193b66b7b08309dfefa7d89db20f8d2a Mon Sep 17 00:00:00 2001 From: Lukas <103962359+L4XB@users.noreply.github.com> Date: Sat, 17 Oct 2026 16:27:49 +0200 Subject: [PATCH] fix(docreader): keep a pipe inside an HTML table cell in that cell (#4433) HTML and MHTML uploads turn tables into GFM through markdownify, which writes a pipe in a cell's text as is. That pipe ends the cell, so a|b became two cells and every later cell in the row moved one column right. Pipes in table cell text are now escaped before the conversion, as `\|`. The escape then has to survive MarkdownTableUtil, which normalizes the converted tables and split rows on every pipe: `| a \| b | c |` came out as a three-column row with the escape turned into `\ |`. It now splits on unescaped pipes only; a pipe after an odd run of backslashes is cell text, as in the Go chunker since #3979. --- docreader/parser/markdown_parser.py | 17 ++++++- docreader/parser/mhtml_parser.py | 17 ++++++- docreader/tests/test_table_cell_pipes.py | 64 ++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 docreader/tests/test_table_cell_pipes.py diff --git a/docreader/parser/markdown_parser.py b/docreader/parser/markdown_parser.py index 937f3c22..bbf6baff 100644 --- a/docreader/parser/markdown_parser.py +++ b/docreader/parser/markdown_parser.py @@ -27,6 +27,9 @@ from docreader.utils import endecode logger = logging.getLogger(__name__) _SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$") +# A backslash escape, or a bare pipe. Scanning both keeps an escaped pipe +# (``\|``) out of the cell delimiters without a lookbehind, in linear time. +_ESCAPE_OR_PIPE = re.compile(r"\\.|\|", re.DOTALL) # Cells of a GFM alignment row as matched by the previous ``align_pattern``. # Looser than ``_SEPARATOR_CELL``: any non-empty run of ``:``/``-`` qualifies. _ALIGNMENT_CELL = re.compile(r"^[:-]+$") @@ -53,11 +56,21 @@ class MarkdownTableUtil: @staticmethod def _split_row_cells(row_line: str) -> List[str]: - """Split a markdown table row into cells, preserving empty cells.""" + """Split a markdown table row into cells, preserving empty cells. + + As in GFM, a pipe escaped as ``\\|`` is text inside its cell rather + than a cell delimiter. + """ inner = row_line.strip() if not inner.startswith("|"): return [] - parts = inner.split("|") + parts: List[str] = [] + start = 0 + for match in _ESCAPE_OR_PIPE.finditer(inner): + if match.group() == "|": + parts.append(inner[start:match.start()]) + start = match.end() + parts.append(inner[start:]) if parts and parts[0].strip() == "": parts = parts[1:] if parts and parts[-1].strip() == "": diff --git a/docreader/parser/mhtml_parser.py b/docreader/parser/mhtml_parser.py index d79ecdd4..113493be 100644 --- a/docreader/parser/mhtml_parser.py +++ b/docreader/parser/mhtml_parser.py @@ -13,7 +13,7 @@ from urllib.parse import unquote, urljoin, urlparse import uuid from typing import Dict -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup, NavigableString from docreader.models.document import Document from docreader.parser.base_parser import BaseParser @@ -289,6 +289,7 @@ class MHTMLParser(BaseParser): self._rewrite_image_sources(soup, image_aliases, base_location) self._unwrap_whitespace_only_inline(soup) text_fallback = soup.get_text(separator="\n", strip=True) + self._escape_table_cell_pipes(soup) markdown_text = md(str(soup), heading_style="ATX") result = self._normalize_markdown(markdown_text) if not result and text_fallback: @@ -326,6 +327,20 @@ class MHTMLParser(BaseParser): if tag.find(True) is None and not tag.get_text().strip(): tag.unwrap() + @staticmethod + def _escape_table_cell_pipes(soup: BeautifulSoup) -> None: + """Escape the ``|`` in table cell text, which markdownify writes as is. + + In the GFM table markdownify writes, a bare pipe in a cell's text ends + the cell, so the row gains a column and every later cell shifts one + column right. + """ + for cell in soup.find_all(["td", "th"]): + for text in cell.find_all(string=True): + # Comments and other special strings keep their own type + if type(text) is NavigableString and "|" in text: + text.replace_with(text.replace("|", "\\|")) + @staticmethod def _normalize_markdown(markdown_text: str) -> str: text = markdown_text.replace("\r\n", "\n").replace("\r", "\n") diff --git a/docreader/tests/test_table_cell_pipes.py b/docreader/tests/test_table_cell_pipes.py new file mode 100644 index 00000000..55154bf2 --- /dev/null +++ b/docreader/tests/test_table_cell_pipes.py @@ -0,0 +1,64 @@ +"""A pipe inside a table cell stays in that cell; GFM escapes it as ``\\|``.""" + +import unittest + +from docreader.parser.html_parser import HTMLParser +from docreader.parser.markdown_parser import MarkdownParser, MarkdownTableUtil + + +class TableCellPipeTest(unittest.TestCase): + def test_format_table_keeps_an_escaped_pipe_in_its_cell(self): + raw = "|Pattern|Matches|\n|---|---|\n|`a\\|b`|a or b|\n|x \\| y|literal|" + + formatted = MarkdownTableUtil().format_table(raw) + + self.assertEqual( + formatted, + "| Pattern | Matches |\n| --- | --- |\n" + "| `a\\|b` | a or b |\n| x \\| y | literal |", + ) + + def test_format_table_ends_a_cell_at_a_pipe_after_an_escaped_backslash(self): + # "\\\\" is an escaped backslash, so the pipe after it is a delimiter + formatted = MarkdownTableUtil().format_table("|a\\\\|b|\n|---|---|") + + self.assertEqual(formatted, "| a\\\\ | b |\n| --- | --- |") + + def test_markdown_upload_keeps_an_escaped_pipe(self): + content = "| Type | Notes |\n| --- | --- |\n| int \\| None | optional |\n" + + document = MarkdownParser( + file_name="types.md", file_type="md" + ).parse_into_text(content.encode()) + + self.assertIn("| int \\| None | optional |", document.content) + + def test_html_upload_escapes_a_pipe_in_a_cell(self): + html = ( + "" + "
ExprMeans
a|brun ls|wc
" + ) + + document = HTMLParser( + file_name="pipes.html", file_type="html" + ).parse_into_text(html.encode()) + + self.assertIn("| Expr | Means |", document.content) + self.assertIn("| a\\|b | run `ls\\|wc` |", document.content) + + def test_html_comment_in_a_cell_stays_hidden(self): + html = ( + "" + "
Name
pen
" + ) + + document = HTMLParser( + file_name="comment.html", file_type="html" + ).parse_into_text(html.encode()) + + self.assertIn("| pen |", document.content) + self.assertNotIn("sku", document.content) + + +if __name__ == "__main__": + unittest.main()