fix(docreader): keep a pipe inside an HTML table cell in that cell (#4433)

HTML and MHTML uploads turn tables into GFM through markdownify, which
writes a pipe in a cell's text as is. That pipe ends the cell, so
<td>a|b</td> became two cells and every later cell in the row moved one
column right. Pipes in table cell text are now escaped before the
conversion, as `\|`.

The escape then has to survive MarkdownTableUtil, which normalizes the
converted tables and split rows on every pipe: `| a \| b | c |` came out
as a three-column row with the escape turned into `\ |`. It now splits
on unescaped pipes only; a pipe after an odd run of backslashes is cell
text, as in the Go chunker since #3979.
main
Lukas 2026-10-17 16:27:49 +02:00 committed by GitHub
parent 345855b59c
commit 578ca1b519
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 95 additions and 3 deletions

View File

@ -27,6 +27,9 @@ from docreader.utils import endecode
logger = logging.getLogger(__name__)
_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$")
# A backslash escape, or a bare pipe. Scanning both keeps an escaped pipe
# (``\|``) out of the cell delimiters without a lookbehind, in linear time.
_ESCAPE_OR_PIPE = re.compile(r"\\.|\|", re.DOTALL)
# Cells of a GFM alignment row as matched by the previous ``align_pattern``.
# Looser than ``_SEPARATOR_CELL``: any non-empty run of ``:``/``-`` qualifies.
_ALIGNMENT_CELL = re.compile(r"^[:-]+$")
@ -53,11 +56,21 @@ class MarkdownTableUtil:
@staticmethod
def _split_row_cells(row_line: str) -> List[str]:
"""Split a markdown table row into cells, preserving empty cells."""
"""Split a markdown table row into cells, preserving empty cells.
As in GFM, a pipe escaped as ``\\|`` is text inside its cell rather
than a cell delimiter.
"""
inner = row_line.strip()
if not inner.startswith("|"):
return []
parts = inner.split("|")
parts: List[str] = []
start = 0
for match in _ESCAPE_OR_PIPE.finditer(inner):
if match.group() == "|":
parts.append(inner[start:match.start()])
start = match.end()
parts.append(inner[start:])
if parts and parts[0].strip() == "":
parts = parts[1:]
if parts and parts[-1].strip() == "":

View File

@ -13,7 +13,7 @@ from urllib.parse import unquote, urljoin, urlparse
import uuid
from typing import Dict
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup, NavigableString
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
@ -289,6 +289,7 @@ class MHTMLParser(BaseParser):
self._rewrite_image_sources(soup, image_aliases, base_location)
self._unwrap_whitespace_only_inline(soup)
text_fallback = soup.get_text(separator="\n", strip=True)
self._escape_table_cell_pipes(soup)
markdown_text = md(str(soup), heading_style="ATX")
result = self._normalize_markdown(markdown_text)
if not result and text_fallback:
@ -326,6 +327,20 @@ class MHTMLParser(BaseParser):
if tag.find(True) is None and not tag.get_text().strip():
tag.unwrap()
@staticmethod
def _escape_table_cell_pipes(soup: BeautifulSoup) -> None:
"""Escape the ``|`` in table cell text, which markdownify writes as is.
In the GFM table markdownify writes, a bare pipe in a cell's text ends
the cell, so the row gains a column and every later cell shifts one
column right.
"""
for cell in soup.find_all(["td", "th"]):
for text in cell.find_all(string=True):
# Comments and other special strings keep their own type
if type(text) is NavigableString and "|" in text:
text.replace_with(text.replace("|", "\\|"))
@staticmethod
def _normalize_markdown(markdown_text: str) -> str:
text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")

View File

@ -0,0 +1,64 @@
"""A pipe inside a table cell stays in that cell; GFM escapes it as ``\\|``."""
import unittest
from docreader.parser.html_parser import HTMLParser
from docreader.parser.markdown_parser import MarkdownParser, MarkdownTableUtil
class TableCellPipeTest(unittest.TestCase):
def test_format_table_keeps_an_escaped_pipe_in_its_cell(self):
raw = "|Pattern|Matches|\n|---|---|\n|`a\\|b`|a or b|\n|x \\| y|literal|"
formatted = MarkdownTableUtil().format_table(raw)
self.assertEqual(
formatted,
"| Pattern | Matches |\n| --- | --- |\n"
"| `a\\|b` | a or b |\n| x \\| y | literal |",
)
def test_format_table_ends_a_cell_at_a_pipe_after_an_escaped_backslash(self):
# "\\\\" is an escaped backslash, so the pipe after it is a delimiter
formatted = MarkdownTableUtil().format_table("|a\\\\|b|\n|---|---|")
self.assertEqual(formatted, "| a\\\\ | b |\n| --- | --- |")
def test_markdown_upload_keeps_an_escaped_pipe(self):
content = "| Type | Notes |\n| --- | --- |\n| int \\| None | optional |\n"
document = MarkdownParser(
file_name="types.md", file_type="md"
).parse_into_text(content.encode())
self.assertIn("| int \\| None | optional |", document.content)
def test_html_upload_escapes_a_pipe_in_a_cell(self):
html = (
"<table><tr><th>Expr</th><th>Means</th></tr>"
"<tr><td>a|b</td><td>run <code>ls|wc</code></td></tr></table>"
)
document = HTMLParser(
file_name="pipes.html", file_type="html"
).parse_into_text(html.encode())
self.assertIn("| Expr | Means |", document.content)
self.assertIn("| a\\|b | run `ls\\|wc` |", document.content)
def test_html_comment_in_a_cell_stays_hidden(self):
html = (
"<table><tr><th>Name</th></tr>"
"<tr><td>pen<!-- sku|42 --></td></tr></table>"
)
document = HTMLParser(
file_name="comment.html", file_type="html"
).parse_into_text(html.encode())
self.assertIn("| pen |", document.content)
self.assertNotIn("sku", document.content)
if __name__ == "__main__":
unittest.main()