From 578ca1b5193b66b7b08309dfefa7d89db20f8d2a Mon Sep 17 00:00:00 2001
From: Lukas <103962359+L4XB@users.noreply.github.com>
Date: Sat, 17 Oct 2026 16:27:49 +0200
Subject: [PATCH] fix(docreader): keep a pipe inside an HTML table cell in that
cell (#4433)
HTML and MHTML uploads turn tables into GFM through markdownify, which
writes a pipe in a cell's text as is. That pipe ends the cell, so
a|b | became two cells and every later cell in the row moved one
column right. Pipes in table cell text are now escaped before the
conversion, as `\|`.
The escape then has to survive MarkdownTableUtil, which normalizes the
converted tables and split rows on every pipe: `| a \| b | c |` came out
as a three-column row with the escape turned into `\ |`. It now splits
on unescaped pipes only; a pipe after an odd run of backslashes is cell
text, as in the Go chunker since #3979.
---
docreader/parser/markdown_parser.py | 17 ++++++-
docreader/parser/mhtml_parser.py | 17 ++++++-
docreader/tests/test_table_cell_pipes.py | 64 ++++++++++++++++++++++++
3 files changed, 95 insertions(+), 3 deletions(-)
create mode 100644 docreader/tests/test_table_cell_pipes.py
diff --git a/docreader/parser/markdown_parser.py b/docreader/parser/markdown_parser.py
index 937f3c22..bbf6baff 100644
--- a/docreader/parser/markdown_parser.py
+++ b/docreader/parser/markdown_parser.py
@@ -27,6 +27,9 @@ from docreader.utils import endecode
logger = logging.getLogger(__name__)
_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$")
+# A backslash escape, or a bare pipe. Scanning both keeps an escaped pipe
+# (``\|``) out of the cell delimiters without a lookbehind, in linear time.
+_ESCAPE_OR_PIPE = re.compile(r"\\.|\|", re.DOTALL)
# Cells of a GFM alignment row as matched by the previous ``align_pattern``.
# Looser than ``_SEPARATOR_CELL``: any non-empty run of ``:``/``-`` qualifies.
_ALIGNMENT_CELL = re.compile(r"^[:-]+$")
@@ -53,11 +56,21 @@ class MarkdownTableUtil:
@staticmethod
def _split_row_cells(row_line: str) -> List[str]:
- """Split a markdown table row into cells, preserving empty cells."""
+ """Split a markdown table row into cells, preserving empty cells.
+
+ As in GFM, a pipe escaped as ``\\|`` is text inside its cell rather
+ than a cell delimiter.
+ """
inner = row_line.strip()
if not inner.startswith("|"):
return []
- parts = inner.split("|")
+ parts: List[str] = []
+ start = 0
+ for match in _ESCAPE_OR_PIPE.finditer(inner):
+ if match.group() == "|":
+ parts.append(inner[start:match.start()])
+ start = match.end()
+ parts.append(inner[start:])
if parts and parts[0].strip() == "":
parts = parts[1:]
if parts and parts[-1].strip() == "":
diff --git a/docreader/parser/mhtml_parser.py b/docreader/parser/mhtml_parser.py
index d79ecdd4..113493be 100644
--- a/docreader/parser/mhtml_parser.py
+++ b/docreader/parser/mhtml_parser.py
@@ -13,7 +13,7 @@ from urllib.parse import unquote, urljoin, urlparse
import uuid
from typing import Dict
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, NavigableString
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
@@ -289,6 +289,7 @@ class MHTMLParser(BaseParser):
self._rewrite_image_sources(soup, image_aliases, base_location)
self._unwrap_whitespace_only_inline(soup)
text_fallback = soup.get_text(separator="\n", strip=True)
+ self._escape_table_cell_pipes(soup)
markdown_text = md(str(soup), heading_style="ATX")
result = self._normalize_markdown(markdown_text)
if not result and text_fallback:
@@ -326,6 +327,20 @@ class MHTMLParser(BaseParser):
if tag.find(True) is None and not tag.get_text().strip():
tag.unwrap()
+ @staticmethod
+ def _escape_table_cell_pipes(soup: BeautifulSoup) -> None:
+ """Escape the ``|`` in table cell text, which markdownify writes as is.
+
+ In the GFM table markdownify writes, a bare pipe in a cell's text ends
+ the cell, so the row gains a column and every later cell shifts one
+ column right.
+ """
+ for cell in soup.find_all(["td", "th"]):
+ for text in cell.find_all(string=True):
+ # Comments and other special strings keep their own type
+ if type(text) is NavigableString and "|" in text:
+ text.replace_with(text.replace("|", "\\|"))
+
@staticmethod
def _normalize_markdown(markdown_text: str) -> str:
text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")
diff --git a/docreader/tests/test_table_cell_pipes.py b/docreader/tests/test_table_cell_pipes.py
new file mode 100644
index 00000000..55154bf2
--- /dev/null
+++ b/docreader/tests/test_table_cell_pipes.py
@@ -0,0 +1,64 @@
+"""A pipe inside a table cell stays in that cell; GFM escapes it as ``\\|``."""
+
+import unittest
+
+from docreader.parser.html_parser import HTMLParser
+from docreader.parser.markdown_parser import MarkdownParser, MarkdownTableUtil
+
+
+class TableCellPipeTest(unittest.TestCase):
+ def test_format_table_keeps_an_escaped_pipe_in_its_cell(self):
+ raw = "|Pattern|Matches|\n|---|---|\n|`a\\|b`|a or b|\n|x \\| y|literal|"
+
+ formatted = MarkdownTableUtil().format_table(raw)
+
+ self.assertEqual(
+ formatted,
+ "| Pattern | Matches |\n| --- | --- |\n"
+ "| `a\\|b` | a or b |\n| x \\| y | literal |",
+ )
+
+ def test_format_table_ends_a_cell_at_a_pipe_after_an_escaped_backslash(self):
+ # "\\\\" is an escaped backslash, so the pipe after it is a delimiter
+ formatted = MarkdownTableUtil().format_table("|a\\\\|b|\n|---|---|")
+
+ self.assertEqual(formatted, "| a\\\\ | b |\n| --- | --- |")
+
+ def test_markdown_upload_keeps_an_escaped_pipe(self):
+ content = "| Type | Notes |\n| --- | --- |\n| int \\| None | optional |\n"
+
+ document = MarkdownParser(
+ file_name="types.md", file_type="md"
+ ).parse_into_text(content.encode())
+
+ self.assertIn("| int \\| None | optional |", document.content)
+
+ def test_html_upload_escapes_a_pipe_in_a_cell(self):
+ html = (
+ "| Expr | Means |
"
+ "| a|b | run ls|wc |
"
+ )
+
+ document = HTMLParser(
+ file_name="pipes.html", file_type="html"
+ ).parse_into_text(html.encode())
+
+ self.assertIn("| Expr | Means |", document.content)
+ self.assertIn("| a\\|b | run `ls\\|wc` |", document.content)
+
+ def test_html_comment_in_a_cell_stays_hidden(self):
+ html = (
+ ""
+ )
+
+ document = HTMLParser(
+ file_name="comment.html", file_type="html"
+ ).parse_into_text(html.encode())
+
+ self.assertIn("| pen |", document.content)
+ self.assertNotIn("sku", document.content)
+
+
+if __name__ == "__main__":
+ unittest.main()