fix(docreader): keep a pipe inside an HTML table cell in that cell (#4433)
HTML and MHTML uploads turn tables into GFM through markdownify, which writes a pipe in a cell's text as is. That pipe ends the cell, so <td>a|b</td> became two cells and every later cell in the row moved one column right. Pipes in table cell text are now escaped before the conversion, as `\|`. The escape then has to survive MarkdownTableUtil, which normalizes the converted tables and split rows on every pipe: `| a \| b | c |` came out as a three-column row with the escape turned into `\ |`. It now splits on unescaped pipes only; a pipe after an odd run of backslashes is cell text, as in the Go chunker since #3979.main
parent
345855b59c
commit
578ca1b519
|
|
@ -27,6 +27,9 @@ from docreader.utils import endecode
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$")
|
||||
# A backslash escape, or a bare pipe. Scanning both keeps an escaped pipe
|
||||
# (``\|``) out of the cell delimiters without a lookbehind, in linear time.
|
||||
_ESCAPE_OR_PIPE = re.compile(r"\\.|\|", re.DOTALL)
|
||||
# Cells of a GFM alignment row as matched by the previous ``align_pattern``.
|
||||
# Looser than ``_SEPARATOR_CELL``: any non-empty run of ``:``/``-`` qualifies.
|
||||
_ALIGNMENT_CELL = re.compile(r"^[:-]+$")
|
||||
|
|
@ -53,11 +56,21 @@ class MarkdownTableUtil:
|
|||
|
||||
@staticmethod
|
||||
def _split_row_cells(row_line: str) -> List[str]:
|
||||
"""Split a markdown table row into cells, preserving empty cells."""
|
||||
"""Split a markdown table row into cells, preserving empty cells.
|
||||
|
||||
As in GFM, a pipe escaped as ``\\|`` is text inside its cell rather
|
||||
than a cell delimiter.
|
||||
"""
|
||||
inner = row_line.strip()
|
||||
if not inner.startswith("|"):
|
||||
return []
|
||||
parts = inner.split("|")
|
||||
parts: List[str] = []
|
||||
start = 0
|
||||
for match in _ESCAPE_OR_PIPE.finditer(inner):
|
||||
if match.group() == "|":
|
||||
parts.append(inner[start:match.start()])
|
||||
start = match.end()
|
||||
parts.append(inner[start:])
|
||||
if parts and parts[0].strip() == "":
|
||||
parts = parts[1:]
|
||||
if parts and parts[-1].strip() == "":
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from urllib.parse import unquote, urljoin, urlparse
|
|||
import uuid
|
||||
from typing import Dict
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from bs4 import BeautifulSoup, NavigableString
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
|
@ -289,6 +289,7 @@ class MHTMLParser(BaseParser):
|
|||
self._rewrite_image_sources(soup, image_aliases, base_location)
|
||||
self._unwrap_whitespace_only_inline(soup)
|
||||
text_fallback = soup.get_text(separator="\n", strip=True)
|
||||
self._escape_table_cell_pipes(soup)
|
||||
markdown_text = md(str(soup), heading_style="ATX")
|
||||
result = self._normalize_markdown(markdown_text)
|
||||
if not result and text_fallback:
|
||||
|
|
@ -326,6 +327,20 @@ class MHTMLParser(BaseParser):
|
|||
if tag.find(True) is None and not tag.get_text().strip():
|
||||
tag.unwrap()
|
||||
|
||||
@staticmethod
|
||||
def _escape_table_cell_pipes(soup: BeautifulSoup) -> None:
|
||||
"""Escape the ``|`` in table cell text, which markdownify writes as is.
|
||||
|
||||
In the GFM table markdownify writes, a bare pipe in a cell's text ends
|
||||
the cell, so the row gains a column and every later cell shifts one
|
||||
column right.
|
||||
"""
|
||||
for cell in soup.find_all(["td", "th"]):
|
||||
for text in cell.find_all(string=True):
|
||||
# Comments and other special strings keep their own type
|
||||
if type(text) is NavigableString and "|" in text:
|
||||
text.replace_with(text.replace("|", "\\|"))
|
||||
|
||||
@staticmethod
|
||||
def _normalize_markdown(markdown_text: str) -> str:
|
||||
text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
"""A pipe inside a table cell stays in that cell; GFM escapes it as ``\\|``."""
|
||||
|
||||
import unittest
|
||||
|
||||
from docreader.parser.html_parser import HTMLParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser, MarkdownTableUtil
|
||||
|
||||
|
||||
class TableCellPipeTest(unittest.TestCase):
|
||||
def test_format_table_keeps_an_escaped_pipe_in_its_cell(self):
|
||||
raw = "|Pattern|Matches|\n|---|---|\n|`a\\|b`|a or b|\n|x \\| y|literal|"
|
||||
|
||||
formatted = MarkdownTableUtil().format_table(raw)
|
||||
|
||||
self.assertEqual(
|
||||
formatted,
|
||||
"| Pattern | Matches |\n| --- | --- |\n"
|
||||
"| `a\\|b` | a or b |\n| x \\| y | literal |",
|
||||
)
|
||||
|
||||
def test_format_table_ends_a_cell_at_a_pipe_after_an_escaped_backslash(self):
|
||||
# "\\\\" is an escaped backslash, so the pipe after it is a delimiter
|
||||
formatted = MarkdownTableUtil().format_table("|a\\\\|b|\n|---|---|")
|
||||
|
||||
self.assertEqual(formatted, "| a\\\\ | b |\n| --- | --- |")
|
||||
|
||||
def test_markdown_upload_keeps_an_escaped_pipe(self):
|
||||
content = "| Type | Notes |\n| --- | --- |\n| int \\| None | optional |\n"
|
||||
|
||||
document = MarkdownParser(
|
||||
file_name="types.md", file_type="md"
|
||||
).parse_into_text(content.encode())
|
||||
|
||||
self.assertIn("| int \\| None | optional |", document.content)
|
||||
|
||||
def test_html_upload_escapes_a_pipe_in_a_cell(self):
|
||||
html = (
|
||||
"<table><tr><th>Expr</th><th>Means</th></tr>"
|
||||
"<tr><td>a|b</td><td>run <code>ls|wc</code></td></tr></table>"
|
||||
)
|
||||
|
||||
document = HTMLParser(
|
||||
file_name="pipes.html", file_type="html"
|
||||
).parse_into_text(html.encode())
|
||||
|
||||
self.assertIn("| Expr | Means |", document.content)
|
||||
self.assertIn("| a\\|b | run `ls\\|wc` |", document.content)
|
||||
|
||||
def test_html_comment_in_a_cell_stays_hidden(self):
|
||||
html = (
|
||||
"<table><tr><th>Name</th></tr>"
|
||||
"<tr><td>pen<!-- sku|42 --></td></tr></table>"
|
||||
)
|
||||
|
||||
document = HTMLParser(
|
||||
file_name="comment.html", file_type="html"
|
||||
).parse_into_text(html.encode())
|
||||
|
||||
self.assertIn("| pen |", document.content)
|
||||
self.assertNotIn("sku", document.content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue