diff --git a/internal/application/service/html_table_normalize_wiring_test.go b/internal/application/service/html_table_normalize_wiring_test.go
new file mode 100644
index 00000000..295776c1
--- /dev/null
+++ b/internal/application/service/html_table_normalize_wiring_test.go
@@ -0,0 +1,107 @@
+package service
+
+import (
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
+)
+
+// Regression guard for the central "normalize inline HTML tables before
+// chunking" wiring. The bug this guards: normalization was only wired into the
+// paddleocr-vl converters, so tables emitted by other engines (e.g. MinerU)
+// reached the chunker as raw single-line
blocks and were force-cut at
+// the absolute size limit.
+func TestHTMLEmbeddedTableWiring_ProducesChunkableOutput(t *testing.T) {
+ input := strings.Join([]string{
+ "# 检测报告",
+ "",
+ "以下是正文段落,用于确认归一化不会破坏普通 markdown。",
+ "",
+ // colspan=1: redundant spans must still be convertible to GFM.
+ ``,
+ "",
+ // colspan=6: a real merge -> cannot be GFM, but must stay splittable.
+ ``,
+ "",
+ "收尾段落。",
+ }, "\n")
+
+ got := docparser.NormalizeHTMLTables(input)
+
+ if strings.Contains(got, `colspan="1"`) {
+ t.Fatalf("redundant colspan=1 table was not normalized:\n%s", got)
+ }
+ if !strings.Contains(got, "|") || !strings.Contains(got, "拉伸强度") {
+ t.Fatalf("expected colspan=1 table to become GFM, got:\n%s", got)
+ }
+ if !strings.Contains(got, " at offset %d is not preceded by a newline:\n%s", abs, got)
+ }
+ i = abs + 1
+ }
+}
+
+func lastCallBefore(src, call, anchor string) int {
+ idx := strings.Index(src, anchor)
+ if idx < 0 {
+ return -1
+ }
+ return strings.LastIndex(src[:idx], call)
+}
+
+// TestHTMLEmbeddedTableWiring_CallSitesPresent is a lightweight static guard:
+// it fails if the central normalization call is removed from a pre-chunk
+// (or pre-image-resolution) location.
+func TestHTMLEmbeddedTableWiring_CallSitesPresent(t *testing.T) {
+ call := "docparser.NormalizeHTMLTables("
+
+ processSrc, err := os.ReadFile("knowledge_process.go")
+ if err != nil {
+ t.Fatalf("read knowledge_process.go: %v", err)
+ }
+ process := string(processSrc)
+ if lastCallBefore(process, call, "s.imageResolver.ResolveAndStore") < 0 {
+ t.Fatalf("knowledge_process.go: %s must run before image resolution", call)
+ }
+ if lastCallBefore(process, call, "chunkCfg := buildSplitterConfigFromChunking") < 0 {
+ t.Fatalf("knowledge_process.go: %s must run before the chunking step", call)
+ }
+
+ createSrc, err := os.ReadFile("knowledge_create.go")
+ if err != nil {
+ t.Fatalf("read knowledge_create.go: %v", err)
+ }
+ if lastCallBefore(string(createSrc), call, "chunkCfg := buildSplitterConfigFromChunking") < 0 {
+ t.Fatalf("knowledge_create.go: %s must run before buildSplitterConfigFromChunking", call)
+ }
+
+ tempSrc, err := os.ReadFile("temporary_document.go")
+ if err != nil {
+ t.Fatalf("read temporary_document.go: %v", err)
+ }
+ if lastCallBefore(string(tempSrc), call, "chunker.Split(") < 0 {
+ t.Fatalf("temporary_document.go: %s must run before chunker.Split", call)
+ }
+
+ previewSrc, err := os.ReadFile("../../handler/chunker_debug.go")
+ if err != nil {
+ t.Fatalf("read chunker_debug.go: %v", err)
+ }
+ if !strings.Contains(string(previewSrc), call) {
+ t.Fatalf("chunker_debug.go is missing the preview %s call", call)
+ }
+}
diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go
index df4f4698..1b19f5a6 100644
--- a/internal/application/service/knowledge_create.go
+++ b/internal/application/service/knowledge_create.go
@@ -1276,6 +1276,11 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
processOverrides, _ := knowledge.ProcessOverrides()
eff := ResolveProcessConfig(kb, processOverrides)
+ // Normalize inline HTML tables before chunking, for the same reason as the
+ // file-processing path: parser/OCR output may embed raw blocks that
+ // the chunker cannot split. Fenced code examples are left untouched.
+ clean = docparser.NormalizeHTMLTables(clean)
+
// Manual content is markdown - chunk directly with Go chunker
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 8cce8685..64859347 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -3689,6 +3689,22 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
convertResult.AudioData = nil
}
+ // Normalize inline HTML tables from any parser engine (MinerU, builtin,
+ // markitdown, paddleocr-vl, ...) into GFM, or row-newline-separated HTML
+ // for merged / non-GFM tables, so the chunker can split them.
+ // Runs before image resolution so
inside HTML tables becomes
+ // markdown the resolver can persist, instead of rewriting data-URI
+ //
tags to `` and then escaping them in a later
+ // HTML→GFM pass. Line endings are normalized first so injected row
+ // breaks are plain LF.
+ sanitizeReadResult(convertResult)
+ if convertResult != nil {
+ convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
+ if convertResult.MarkdownContent != "" {
+ convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent)
+ }
+ }
+
// Step 2: Store images and update markdown references
var storedImages []docparser.StoredImage
@@ -3719,11 +3735,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
logger.Infof(ctx, "Resolved %d total images for knowledge %s", len(storedImages), knowledge.ID)
}
- // Step 3: Split into chunks using Go chunker. Browser textareas normalize
- // pasted content to LF, so normalize uploaded source text before calculating
- // chunk boundaries as well.
- sanitizeReadResult(convertResult)
- convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
+ // Step 3: Split into chunks using Go chunker. Line endings and inline
+ // HTML tables were normalized before image resolution above.
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
processOpts := ProcessChunksOptions{
diff --git a/internal/application/service/ocr_sanitizer.go b/internal/application/service/ocr_sanitizer.go
index dfa5fb6e..fba14b03 100644
--- a/internal/application/service/ocr_sanitizer.go
+++ b/internal/application/service/ocr_sanitizer.go
@@ -5,6 +5,7 @@ import (
"strings"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser"
)
var (
@@ -49,6 +50,13 @@ func sanitizeOCRText(raw string) string {
}
}
+ // Convert inline HTML blocks to GFM tables. This is a safe no-op
+ // when no is present, so it runs unconditionally: a markdown body
+ // with an embedded HTML table does not satisfy looksLikeHTML (it neither
+ // starts with a tag nor is dominated by tag characters), yet leaving the
+ // raw markup in place makes the chunker split inside table rows.
+ text = docparser.NormalizeHTMLTables(text)
+
if isKnownEmptyReply(text) {
return ""
}
diff --git a/internal/application/service/ocr_sanitizer_test.go b/internal/application/service/ocr_sanitizer_test.go
index b6a91c9c..1281bc28 100644
--- a/internal/application/service/ocr_sanitizer_test.go
+++ b/internal/application/service/ocr_sanitizer_test.go
@@ -1,6 +1,10 @@
package service
-import "testing"
+import (
+ "regexp"
+ "strings"
+ "testing"
+)
func TestSanitizeOCRText(t *testing.T) {
tests := []struct {
@@ -187,6 +191,46 @@ func TestLooksLikeHTML(t *testing.T) {
}
}
+// TestSanitizeOCRText_ConvertsInlineHTMLTable guards against a regression where
+// an HTML embedded in a markdown body survived sanitization untouched:
+// the mixed content does not satisfy looksLikeHTML, so the old HTML-to-markdown
+// path never ran and the raw markup reached the chunker.
+func TestSanitizeOCRText_ConvertsInlineHTMLTable(t *testing.T) {
+ para := "本季度公司整体经营情况保持稳定,营收与利润两项核心指标均实现同比增长。" +
+ "为了便于管理层快速了解经营成果,下表汇总了报告期内的主要财务数据," +
+ "供后续经营分析、预算编制以及年度考核等工作参考使用," +
+ "请结合实际业务情况综合判断,切勿脱离业务背景单独解读其中任何一项数字。"
+ tail := "以上数据均来自财务部门审核后的正式报表,统计口径与上一报告期保持一致," +
+ "未发生会计政策变更或追溯调整。如有疑问请与财务部门联系确认," +
+ "最终解释权归公司财务部门所有。"
+ input := "# 报告\n\n" + para + "\n\n" +
+ `| 指标 | 数值 |
` +
+ `| 营收 | 10亿 |
` +
+ `| 利润 | 2.3亿 |
` +
+ "\n\n" + tail + "\n\n"
+
+ if looksLikeHTML(input) {
+ t.Fatalf("test input unexpectedly looks like HTML; the regression test must exercise the mixed-content path")
+ }
+
+ got := sanitizeOCRText(input)
+
+ if strings.Contains(got, " best {
+ best = n
+ }
+ }
+ return best
+}
+
+// TestSplitText_OCRStrayBracketDoesNotSwallowDocument is the core regression:
+// before the fix the malformed '[' turned the whole document into a single
+// protected span (~1700 runes) and SplitText returned one oversized chunk.
+func TestSplitText_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) {
+ doc := buildOCRStrayBracketDoc()
+ if !strings.Contains(doc, "[公章】") || !strings.Contains(doc, "](来源)") {
+ t.Fatalf("fixture missing malformed-bracket markers")
+ }
+ const chunkSize = 128
+ cfg := SplitterConfig{ChunkSize: chunkSize, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", "。"}}
+
+ chunks := SplitText(doc, cfg)
+ t.Logf("doc=%d runes chunkSize=%d -> chunks=%d max=%d runes",
+ len([]rune(doc)), chunkSize, len(chunks), maxChunkRunes(chunks))
+ if len(chunks) < 2 {
+ t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks))
+ }
+ if got := maxChunkRunes(chunks); got > 2*chunkSize {
+ t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize)
+ }
+}
+
+// TestSplit_OCRStrayBracketDoesNotSwallowDocument exercises the strategy-aware
+// entry point with the legacy tier pinned, so the protected-pattern behaviour
+// is what is under test rather than the profiler's tier selection.
+func TestSplit_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) {
+ doc := buildOCRStrayBracketDoc()
+ const chunkSize = 128
+ cfg := SplitterConfig{
+ ChunkSize: chunkSize,
+ ChunkOverlap: 0,
+ Separators: []string{"\n\n", "\n", "。"},
+ Strategy: StrategyLegacy,
+ }
+
+ chunks := Split(doc, cfg)
+ if len(chunks) < 2 {
+ t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks))
+ }
+ if got := maxChunkRunes(chunks); got > 2*chunkSize {
+ t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize)
+ }
+}
+
+// TestProtectedSpans_NoCrossLineLinkOrImage asserts no protected span may
+// contain a newline for the malformed document: link text and destination
+// cannot span a blank line (CommonMark).
+func TestProtectedSpans_NoCrossLineLinkOrImage(t *testing.T) {
+ doc := buildOCRStrayBracketDoc()
+ for _, s := range protectedSpans(doc) {
+ seg := doc[s.start:s.end]
+ if strings.Contains(seg, "\n") {
+ t.Fatalf("protected span spans a newline (%d runes): %q", len([]rune(seg)), seg)
+ }
+ }
+}
+
+// TestProtectedSpans_MarkdownLinksAndImagesStillProtected is the positive
+// guard: the newline-bounded patterns must keep recognising well-formed
+// single-line Markdown links and images.
+func TestProtectedSpans_MarkdownLinksAndImagesStillProtected(t *testing.T) {
+ cases := []string{
+ "",
+ "[文字](http://x.com)",
+ `[link](url "title")`,
+ "",
+ }
+ for _, c := range cases {
+ spans := protectedSpans(c)
+ if len(spans) == 0 {
+ t.Errorf("expected %q to be protected, got no span", c)
+ continue
+ }
+ if spans[0].start != 0 || spans[0].end != len(c) {
+ t.Errorf("%q: protected span = [%d,%d), want [0,%d)", c, spans[0].start, spans[0].end, len(c))
+ }
+ }
+}
+
+// TestProtectedSpans_LiteralCrossLineExample documents that an isolated
+// malformed bracket pair with NO complete '](...)' construct is never treated
+// as a protected link/image region.
+func TestProtectedSpans_MultilineLinkIntentionallyUnprotected(t *testing.T) {
+ // CommonMark allows a single soft line break in link text. The protected
+ // patterns still refuse it so a stray OCR '[' cannot swallow a paragraph.
+ doc := "[link\ntext](http://example.com)"
+ if spans := protectedSpans(doc); len(spans) != 0 {
+ t.Fatalf("expected multiline link to stay unprotected, got %v", spans)
+ }
+}
+
+func TestProtectedSpans_LiteralCrossLineExample(t *testing.T) {
+ doc := "[公章】\n\n第一段。\n\n第二段。\n\n第三段(北京)"
+ if spans := protectedSpans(doc); len(spans) != 0 {
+ t.Fatalf("expected no protected span, got %v", spans)
+ }
+}
diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go
index ff415eed..69ff38e4 100644
--- a/internal/infrastructure/chunker/splitter.go
+++ b/internal/infrastructure/chunker/splitter.go
@@ -114,10 +114,18 @@ func DefaultConfig() SplitterConfig {
}
// protectedPatterns are regex patterns for content that must not be split.
+//
+// The Markdown link/image patterns deliberately exclude '\n' and bound the
+// link text / destination length: CommonMark forbids them from spanning a
+// blank line, and the previous unbounded [^\]]* / [^)]+ let a stray '[' left
+// behind by OCR swallow whole paragraphs as one "protected" atomic span,
+// defeating chunking entirely.
var protectedPatterns = []*regexp.Regexp{
- regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
- regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images
- regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links
+ regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
+ // Markdown images / links: single line, bounded so a stray OCR '['
+ // cannot swallow a paragraph (CommonMark forbids blank-line spans).
+ regexp.MustCompile(`!\[[^\]\n]{0,200}\]\([^)\n]{1,500}\)`),
+ regexp.MustCompile(`\[[^\]\n]{1,200}\]\([^)\n]{1,500}\)`),
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator
regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows
regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks
diff --git a/internal/infrastructure/docparser/html_table_normalizer.go b/internal/infrastructure/docparser/html_table_normalizer.go
index ad55b196..8851c2ba 100644
--- a/internal/infrastructure/docparser/html_table_normalizer.go
+++ b/internal/infrastructure/docparser/html_table_normalizer.go
@@ -15,6 +15,10 @@ var (
// block, the form OCR/layout engines such as PaddleOCR-VL emit tables in.
htmlTableBlockPattern = regexp.MustCompile(`(?is)`)
+ // fencedCodeBlockPattern matches CommonMark fenced code blocks so HTML
+ // examples inside them are left untouched.
+ fencedCodeBlockPattern = regexp.MustCompile("(?s)(?:```|~~~)[^\\n]*\\n.*?(?:```|~~~)")
+
// htmlLayoutAttrPattern matches presentational HTML attributes that carry
// no semantic value (text-align styles, CSS classes, sizing). Structural
// attributes like rowspan/colspan are intentionally excluded.
@@ -22,28 +26,45 @@ var (
`(?is)\s+(?:style|class|align|valign|width|height|bgcolor)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)`,
)
- // htmlSpanAttrPattern detects rowspan/colspan, which Markdown tables cannot
- // represent; such tables keep their HTML form (attributes stripped) instead.
- htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\b`)
+ // htmlSpanAttrPattern detects rowspan/colspan values greater than 1, which
+ // Markdown tables cannot represent; such tables keep their HTML form
+ // (attributes stripped) instead. Span values of 1 (and the invalid 0) do
+ // not merge anything and must stay convertible.
+ htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\s*=\s*["']?(?:[2-9]|\d{2,})`)
- // markdownTableSeparatorPattern matches the |---|---| delimiter row that a
- // valid GFM table must contain.
- markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)+\|?\s*$`)
+ // htmlTableRowPattern matches the opening tag of each table row, used
+ // to put every row on its own line so the chunker can split degraded HTML
+ // tables at "\n" boundaries.
+ htmlTableRowPattern = regexp.MustCompile(`(?i)
blocks embedded in OCR
+// NormalizeHTMLTables rewrites inline HTML blocks embedded in OCR
// markdown output. PaddleOCR-VL emits tables as HTML with per-cell text-align
// styles, which (1) waste tokens on layout markup and (2) are not recognized
// by the chunker's table-protection logic, so large tables get split mid-row.
//
// Each table block is converted to a GFM Markdown table when possible. Tables
-// that use rowspan/colspan (which Markdown cannot express) fall back to having
-// their presentational attributes stripped so they stay intact as HTML.
-func normalizeHTMLTables(md string) string {
+// that use rowspan/colspan (which Markdown cannot express), or that the
+// converter cannot turn into a valid GFM table (cell
, nested lists, …),
+// fall back to presentational-attribute stripping plus one row per line so
+// the chunker can still split on "\n". Tables inside fenced code blocks are
+// left unchanged.
+func NormalizeHTMLTables(md string) string {
if !strings.Contains(strings.ToLower(md), "= s[0] && pos < s[1] {
+ return true
}
- // Pad with blank lines so the Markdown table is a standalone block that
- // the chunker recognizes (and protects) as a table.
- return "\n\n" + converted + "\n\n"
- })
+ }
+ return false
}
// stripHTMLLayoutAttrs removes presentational attributes from an HTML fragment
@@ -75,3 +133,27 @@ func normalizeHTMLTables(md string) string {
func stripHTMLLayoutAttrs(html string) string {
return htmlLayoutAttrPattern.ReplaceAllString(html, "")
}
+
+// splitHTMLTableRows puts each table row on its own line. The chunker splits
+// on "\n", so a single-line HTML table would otherwise be unsplittable and get
+// force-cut at the absolute max size. Whitespace between tags is insignificant
+// in HTML, and newlines are only inserted before (never inside a tag).
+// A
that already starts on its own line is left alone.
+func splitHTMLTableRows(block string) string {
+ locs := htmlTableRowPattern.FindAllStringIndex(block, -1)
+ if len(locs) == 0 {
+ return strings.TrimSpace(block)
+ }
+ var b strings.Builder
+ last := 0
+ for _, loc := range locs {
+ b.WriteString(block[last:loc[0]])
+ if loc[0] == 0 || block[loc[0]-1] != '\n' {
+ b.WriteByte('\n')
+ }
+ b.WriteString(block[loc[0]:loc[1]])
+ last = loc[1]
+ }
+ b.WriteString(block[last:])
+ return strings.TrimSpace(b.String())
+}
diff --git a/internal/infrastructure/docparser/html_table_normalizer_test.go b/internal/infrastructure/docparser/html_table_normalizer_test.go
index 8ae31d59..19b036bb 100644
--- a/internal/infrastructure/docparser/html_table_normalizer_test.go
+++ b/internal/infrastructure/docparser/html_table_normalizer_test.go
@@ -16,7 +16,7 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) {
结尾。`
- got := normalizeHTMLTables(input)
+ got := NormalizeHTMLTables(input)
if strings.Contains(got, "` +
+ `| 指标 | ` +
+ `数值 |
` +
+ `| 营收 | ` +
+ `10亿 |
`
+
+ got := NormalizeHTMLTables(input)
+
+ if strings.Contains(got, "1 / rowspan>1) cannot be GFM, so the table stays
+ // HTML — but each row must start on its own line so the chunker can split it.
+ input := ``
+
+ got := NormalizeHTMLTables(input)
+
+ if !strings.Contains(got, " to start on a new line, got:\n%q", got)
+ }
+ if !strings.Contains(got, "\n\n\n\n") {
+ t.Fatalf("expected block to be padded at the tail with blank lines, got:\n%q", got)
+ }
+ if n := strings.Count(got, "\n at offset %d is not preceded by a newline:\n%s", abs, got)
+ }
+ i = abs + 1
+ }
+}
+
func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) {
// rowspan/colspan cannot be expressed in Markdown, so the table stays HTML
- // but its presentational attributes are stripped.
- input := `| 合计 |
` +
+ // but its presentational attributes are stripped and rows become splittable.
+ input := ``
- got := normalizeHTMLTables(input)
+ got := NormalizeHTMLTables(input)
if !strings.Contains(got, " inside a cell makes the table plugin give up (it emits a newline).
+ // The HTML fallback must still put each on its own line.
+ input := `指标 名称 | 数值 单位 |
` +
+ `拉伸强度 MPa | 合格 A级 |
`
+
+ got := NormalizeHTMLTables(input)
+
+ if !strings.Contains(got, " table to remain HTML rather than flattened text, got:\n%s", got)
+ }
+ assertHTMLRowsSplittable(t, got)
+ for _, want := range []string{"指标", "名称", "拉伸强度", "合格"} {
+ if !strings.Contains(got, want) {
+ t.Fatalf("expected fallback HTML to retain %q, got:\n%s", want, got)
+ }
+ }
+}
+
+func TestNormalizeHTMLTables_ListInCellFallsBackToSplittableHTML(t *testing.T) {
+ input := ``
+
+ got := NormalizeHTMLTables(input)
+
+ if !strings.Contains(got, "`
+
+ got := NormalizeHTMLTables(input)
+
+ if strings.Contains(got, "`
+
+ got := NormalizeHTMLTables(input)
+
+ if strings.Contains(got, "|  | 说明 |
`
+
+ got := NormalizeHTMLTables(input)
+
+ if strings.Contains(got, `!\[`) {
+ t.Fatalf("markdown image syntax escaped:\n%s", got)
+ }
+ if !strings.Contains(got, "") {
+ t.Fatalf("expected markdown image to survive conversion, got:\n%s", got)
+ }
+}
+
+func TestNormalizeHTMLTables_CodeFenceTableLeftAlone(t *testing.T) {
+ input := "示例:\n\n```html\n\n```\n"
+
+ got := NormalizeHTMLTables(input)
+
+ if got != input {
+ t.Fatalf("expected fenced HTML table to be unchanged, got:\n%s", got)
+ }
+}
+
+func TestNormalizeHTMLTables_SpanTableIdempotent(t *testing.T) {
+ input := ``
+ once := NormalizeHTMLTables(input)
+ twice := NormalizeHTMLTables(once)
+ if once != twice {
+ t.Fatalf("NormalizeHTMLTables is not idempotent for span tables:\n1=%q\n2=%q", once, twice)
}
}
diff --git a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
index 98cfc013..c05c9561 100644
--- a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
+++ b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go
@@ -78,7 +78,7 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques
return nil, fmt.Errorf("PaddleOCR-VL Cloud fetch results: %w", err)
}
- mdContent = normalizeHTMLTables(mdContent)
+ mdContent = NormalizeHTMLTables(mdContent)
imageRefs := c.downloadImages(mdContent, imagesURL)
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
diff --git a/internal/infrastructure/docparser/paddleocr_vl_converter.go b/internal/infrastructure/docparser/paddleocr_vl_converter.go
index 313d3b74..f49e1df5 100644
--- a/internal/infrastructure/docparser/paddleocr_vl_converter.go
+++ b/internal/infrastructure/docparser/paddleocr_vl_converter.go
@@ -65,7 +65,7 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*
// wastes tokens and defeats the chunker's table-protection logic. Convert
// them to Markdown tables (or strip layout attributes when conversion is
// not possible) before downstream processing.
- mdContent = normalizeHTMLTables(mdContent)
+ mdContent = NormalizeHTMLTables(mdContent)
imageRefs, mdContent := c.processImages(mdContent, imagesB64)
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)