diff --git a/internal/application/service/html_table_normalize_wiring_test.go b/internal/application/service/html_table_normalize_wiring_test.go new file mode 100644 index 00000000..295776c1 --- /dev/null +++ b/internal/application/service/html_table_normalize_wiring_test.go @@ -0,0 +1,107 @@ +package service + +import ( + "os" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/infrastructure/docparser" +) + +// Regression guard for the central "normalize inline HTML tables before +// chunking" wiring. The bug this guards: normalization was only wired into the +// paddleocr-vl converters, so tables emitted by other engines (e.g. MinerU) +// reached the chunker as raw single-line blocks and were force-cut at +// the absolute size limit. +func TestHTMLEmbeddedTableWiring_ProducesChunkableOutput(t *testing.T) { + input := strings.Join([]string{ + "# 检测报告", + "", + "以下是正文段落,用于确认归一化不会破坏普通 markdown。", + "", + // colspan=1: redundant spans must still be convertible to GFM. + `
` + + `` + + `
项目结果
拉伸强度合格
`, + "", + // colspan=6: a real merge -> cannot be GFM, but must stay splittable. + `` + + `
汇总
备注
`, + "", + "收尾段落。", + }, "\n") + + got := docparser.NormalizeHTMLTables(input) + + if strings.Contains(got, `colspan="1"`) { + t.Fatalf("redundant colspan=1 table was not normalized:\n%s", got) + } + if !strings.Contains(got, "|") || !strings.Contains(got, "拉伸强度") { + t.Fatalf("expected colspan=1 table to become GFM, got:\n%s", got) + } + if !strings.Contains(got, " at offset %d is not preceded by a newline:\n%s", abs, got) + } + i = abs + 1 + } +} + +func lastCallBefore(src, call, anchor string) int { + idx := strings.Index(src, anchor) + if idx < 0 { + return -1 + } + return strings.LastIndex(src[:idx], call) +} + +// TestHTMLEmbeddedTableWiring_CallSitesPresent is a lightweight static guard: +// it fails if the central normalization call is removed from a pre-chunk +// (or pre-image-resolution) location. +func TestHTMLEmbeddedTableWiring_CallSitesPresent(t *testing.T) { + call := "docparser.NormalizeHTMLTables(" + + processSrc, err := os.ReadFile("knowledge_process.go") + if err != nil { + t.Fatalf("read knowledge_process.go: %v", err) + } + process := string(processSrc) + if lastCallBefore(process, call, "s.imageResolver.ResolveAndStore") < 0 { + t.Fatalf("knowledge_process.go: %s must run before image resolution", call) + } + if lastCallBefore(process, call, "chunkCfg := buildSplitterConfigFromChunking") < 0 { + t.Fatalf("knowledge_process.go: %s must run before the chunking step", call) + } + + createSrc, err := os.ReadFile("knowledge_create.go") + if err != nil { + t.Fatalf("read knowledge_create.go: %v", err) + } + if lastCallBefore(string(createSrc), call, "chunkCfg := buildSplitterConfigFromChunking") < 0 { + t.Fatalf("knowledge_create.go: %s must run before buildSplitterConfigFromChunking", call) + } + + tempSrc, err := os.ReadFile("temporary_document.go") + if err != nil { + t.Fatalf("read temporary_document.go: %v", err) + } + if lastCallBefore(string(tempSrc), call, "chunker.Split(") < 0 { + t.Fatalf("temporary_document.go: %s must run before chunker.Split", call) + } + + previewSrc, err := os.ReadFile("../../handler/chunker_debug.go") + if err != nil { + t.Fatalf("read chunker_debug.go: %v", err) + } + if !strings.Contains(string(previewSrc), call) { + t.Fatalf("chunker_debug.go is missing the preview %s call", call) + } +} diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index df4f4698..1b19f5a6 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -1276,6 +1276,11 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context, processOverrides, _ := knowledge.ProcessOverrides() eff := ResolveProcessConfig(kb, processOverrides) + // Normalize inline HTML tables before chunking, for the same reason as the + // file-processing path: parser/OCR output may embed raw blocks that + // the chunker cannot split. Fenced code examples are left untouched. + clean = docparser.NormalizeHTMLTables(clean) + // Manual content is markdown - chunk directly with Go chunker chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig) diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 8cce8685..64859347 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3689,6 +3689,22 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e convertResult.AudioData = nil } + // Normalize inline HTML tables from any parser engine (MinerU, builtin, + // markitdown, paddleocr-vl, ...) into GFM, or row-newline-separated HTML + // for merged / non-GFM tables, so the chunker can split them. + // Runs before image resolution so inside HTML tables becomes + // markdown the resolver can persist, instead of rewriting data-URI + // tags to `![...](...)` and then escaping them in a later + // HTML→GFM pass. Line endings are normalized first so injected row + // breaks are plain LF. + sanitizeReadResult(convertResult) + if convertResult != nil { + convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent) + if convertResult.MarkdownContent != "" { + convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent) + } + } + // Step 2: Store images and update markdown references var storedImages []docparser.StoredImage @@ -3719,11 +3735,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e logger.Infof(ctx, "Resolved %d total images for knowledge %s", len(storedImages), knowledge.ID) } - // Step 3: Split into chunks using Go chunker. Browser textareas normalize - // pasted content to LF, so normalize uploaded source text before calculating - // chunk boundaries as well. - sanitizeReadResult(convertResult) - convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent) + // Step 3: Split into chunks using Go chunker. Line endings and inline + // HTML tables were normalized before image resolution above. chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig) processOpts := ProcessChunksOptions{ diff --git a/internal/application/service/ocr_sanitizer.go b/internal/application/service/ocr_sanitizer.go index dfa5fb6e..fba14b03 100644 --- a/internal/application/service/ocr_sanitizer.go +++ b/internal/application/service/ocr_sanitizer.go @@ -5,6 +5,7 @@ import ( "strings" htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2" + "github.com/Tencent/WeKnora/internal/infrastructure/docparser" ) var ( @@ -49,6 +50,13 @@ func sanitizeOCRText(raw string) string { } } + // Convert inline HTML
blocks to GFM tables. This is a safe no-op + // when no
is present, so it runs unconditionally: a markdown body + // with an embedded HTML table does not satisfy looksLikeHTML (it neither + // starts with a tag nor is dominated by tag characters), yet leaving the + // raw markup in place makes the chunker split inside table rows. + text = docparser.NormalizeHTMLTables(text) + if isKnownEmptyReply(text) { return "" } diff --git a/internal/application/service/ocr_sanitizer_test.go b/internal/application/service/ocr_sanitizer_test.go index b6a91c9c..1281bc28 100644 --- a/internal/application/service/ocr_sanitizer_test.go +++ b/internal/application/service/ocr_sanitizer_test.go @@ -1,6 +1,10 @@ package service -import "testing" +import ( + "regexp" + "strings" + "testing" +) func TestSanitizeOCRText(t *testing.T) { tests := []struct { @@ -187,6 +191,46 @@ func TestLooksLikeHTML(t *testing.T) { } } +// TestSanitizeOCRText_ConvertsInlineHTMLTable guards against a regression where +// an HTML
embedded in a markdown body survived sanitization untouched: +// the mixed content does not satisfy looksLikeHTML, so the old HTML-to-markdown +// path never ran and the raw
markup reached the chunker. +func TestSanitizeOCRText_ConvertsInlineHTMLTable(t *testing.T) { + para := "本季度公司整体经营情况保持稳定,营收与利润两项核心指标均实现同比增长。" + + "为了便于管理层快速了解经营成果,下表汇总了报告期内的主要财务数据," + + "供后续经营分析、预算编制以及年度考核等工作参考使用," + + "请结合实际业务情况综合判断,切勿脱离业务背景单独解读其中任何一项数字。" + tail := "以上数据均来自财务部门审核后的正式报表,统计口径与上一报告期保持一致," + + "未发生会计政策变更或追溯调整。如有疑问请与财务部门联系确认," + + "最终解释权归公司财务部门所有。" + input := "# 报告\n\n" + para + "\n\n" + + `
` + + `` + + `
指标数值
营收10亿
利润2.3亿
` + + "\n\n" + tail + "\n\n" + + if looksLikeHTML(input) { + t.Fatalf("test input unexpectedly looks like HTML; the regression test must exercise the mixed-content path") + } + + got := sanitizeOCRText(input) + + if strings.Contains(got, " best { + best = n + } + } + return best +} + +// TestSplitText_OCRStrayBracketDoesNotSwallowDocument is the core regression: +// before the fix the malformed '[' turned the whole document into a single +// protected span (~1700 runes) and SplitText returned one oversized chunk. +func TestSplitText_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) { + doc := buildOCRStrayBracketDoc() + if !strings.Contains(doc, "[公章】") || !strings.Contains(doc, "](来源)") { + t.Fatalf("fixture missing malformed-bracket markers") + } + const chunkSize = 128 + cfg := SplitterConfig{ChunkSize: chunkSize, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", "。"}} + + chunks := SplitText(doc, cfg) + t.Logf("doc=%d runes chunkSize=%d -> chunks=%d max=%d runes", + len([]rune(doc)), chunkSize, len(chunks), maxChunkRunes(chunks)) + if len(chunks) < 2 { + t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks)) + } + if got := maxChunkRunes(chunks); got > 2*chunkSize { + t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize) + } +} + +// TestSplit_OCRStrayBracketDoesNotSwallowDocument exercises the strategy-aware +// entry point with the legacy tier pinned, so the protected-pattern behaviour +// is what is under test rather than the profiler's tier selection. +func TestSplit_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) { + doc := buildOCRStrayBracketDoc() + const chunkSize = 128 + cfg := SplitterConfig{ + ChunkSize: chunkSize, + ChunkOverlap: 0, + Separators: []string{"\n\n", "\n", "。"}, + Strategy: StrategyLegacy, + } + + chunks := Split(doc, cfg) + if len(chunks) < 2 { + t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks)) + } + if got := maxChunkRunes(chunks); got > 2*chunkSize { + t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize) + } +} + +// TestProtectedSpans_NoCrossLineLinkOrImage asserts no protected span may +// contain a newline for the malformed document: link text and destination +// cannot span a blank line (CommonMark). +func TestProtectedSpans_NoCrossLineLinkOrImage(t *testing.T) { + doc := buildOCRStrayBracketDoc() + for _, s := range protectedSpans(doc) { + seg := doc[s.start:s.end] + if strings.Contains(seg, "\n") { + t.Fatalf("protected span spans a newline (%d runes): %q", len([]rune(seg)), seg) + } + } +} + +// TestProtectedSpans_MarkdownLinksAndImagesStillProtected is the positive +// guard: the newline-bounded patterns must keep recognising well-formed +// single-line Markdown links and images. +func TestProtectedSpans_MarkdownLinksAndImagesStillProtected(t *testing.T) { + cases := []string{ + "![alt](http://x/y.png)", + "[文字](http://x.com)", + `[link](url "title")`, + "![](resource://abc)", + } + for _, c := range cases { + spans := protectedSpans(c) + if len(spans) == 0 { + t.Errorf("expected %q to be protected, got no span", c) + continue + } + if spans[0].start != 0 || spans[0].end != len(c) { + t.Errorf("%q: protected span = [%d,%d), want [0,%d)", c, spans[0].start, spans[0].end, len(c)) + } + } +} + +// TestProtectedSpans_LiteralCrossLineExample documents that an isolated +// malformed bracket pair with NO complete '](...)' construct is never treated +// as a protected link/image region. +func TestProtectedSpans_MultilineLinkIntentionallyUnprotected(t *testing.T) { + // CommonMark allows a single soft line break in link text. The protected + // patterns still refuse it so a stray OCR '[' cannot swallow a paragraph. + doc := "[link\ntext](http://example.com)" + if spans := protectedSpans(doc); len(spans) != 0 { + t.Fatalf("expected multiline link to stay unprotected, got %v", spans) + } +} + +func TestProtectedSpans_LiteralCrossLineExample(t *testing.T) { + doc := "[公章】\n\n第一段。\n\n第二段。\n\n第三段(北京)" + if spans := protectedSpans(doc); len(spans) != 0 { + t.Fatalf("expected no protected span, got %v", spans) + } +} diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index ff415eed..69ff38e4 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -114,10 +114,18 @@ func DefaultConfig() SplitterConfig { } // protectedPatterns are regex patterns for content that must not be split. +// +// The Markdown link/image patterns deliberately exclude '\n' and bound the +// link text / destination length: CommonMark forbids them from spanning a +// blank line, and the previous unbounded [^\]]* / [^)]+ let a stray '[' left +// behind by OCR swallow whole paragraphs as one "protected" atomic span, +// defeating chunking entirely. var protectedPatterns = []*regexp.Regexp{ - regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math - regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images - regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links + regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math + // Markdown images / links: single line, bounded so a stray OCR '[' + // cannot swallow a paragraph (CommonMark forbids blank-line spans). + regexp.MustCompile(`!\[[^\]\n]{0,200}\]\([^)\n]{1,500}\)`), + regexp.MustCompile(`\[[^\]\n]{1,200}\]\([^)\n]{1,500}\)`), regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks diff --git a/internal/infrastructure/docparser/html_table_normalizer.go b/internal/infrastructure/docparser/html_table_normalizer.go index ad55b196..8851c2ba 100644 --- a/internal/infrastructure/docparser/html_table_normalizer.go +++ b/internal/infrastructure/docparser/html_table_normalizer.go @@ -15,6 +15,10 @@ var ( // block, the form OCR/layout engines such as PaddleOCR-VL emit tables in. htmlTableBlockPattern = regexp.MustCompile(`(?is)]*>.*?`) + // fencedCodeBlockPattern matches CommonMark fenced code blocks so HTML + // examples inside them are left untouched. + fencedCodeBlockPattern = regexp.MustCompile("(?s)(?:```|~~~)[^\\n]*\\n.*?(?:```|~~~)") + // htmlLayoutAttrPattern matches presentational HTML attributes that carry // no semantic value (text-align styles, CSS classes, sizing). Structural // attributes like rowspan/colspan are intentionally excluded. @@ -22,28 +26,45 @@ var ( `(?is)\s+(?:style|class|align|valign|width|height|bgcolor)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)`, ) - // htmlSpanAttrPattern detects rowspan/colspan, which Markdown tables cannot - // represent; such tables keep their HTML form (attributes stripped) instead. - htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\b`) + // htmlSpanAttrPattern detects rowspan/colspan values greater than 1, which + // Markdown tables cannot represent; such tables keep their HTML form + // (attributes stripped) instead. Span values of 1 (and the invalid 0) do + // not merge anything and must stay convertible. + htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\s*=\s*["']?(?:[2-9]|\d{2,})`) - // markdownTableSeparatorPattern matches the |---|---| delimiter row that a - // valid GFM table must contain. - markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)+\|?\s*$`) + // htmlTableRowPattern matches the opening tag of each table row, used + // to put every row on its own line so the chunker can split degraded HTML + // tables at "\n" boundaries. + htmlTableRowPattern = regexp.MustCompile(`(?i) blocks embedded in OCR +// NormalizeHTMLTables rewrites inline HTML
blocks embedded in OCR // markdown output. PaddleOCR-VL emits tables as HTML with per-cell text-align // styles, which (1) waste tokens on layout markup and (2) are not recognized // by the chunker's table-protection logic, so large tables get split mid-row. // // Each table block is converted to a GFM Markdown table when possible. Tables -// that use rowspan/colspan (which Markdown cannot express) fall back to having -// their presentational attributes stripped so they stay intact as HTML. -func normalizeHTMLTables(md string) string { +// that use rowspan/colspan (which Markdown cannot express), or that the +// converter cannot turn into a valid GFM table (cell
, nested lists, …), +// fall back to presentational-attribute stripping plus one row per line so +// the chunker can still split on "\n". Tables inside fenced code blocks are +// left unchanged. +func NormalizeHTMLTables(md string) string { if !strings.Contains(strings.ToLower(md), "= s[0] && pos < s[1] { + return true } - // Pad with blank lines so the Markdown table is a standalone block that - // the chunker recognizes (and protects) as a table. - return "\n\n" + converted + "\n\n" - }) + } + return false } // stripHTMLLayoutAttrs removes presentational attributes from an HTML fragment @@ -75,3 +133,27 @@ func normalizeHTMLTables(md string) string { func stripHTMLLayoutAttrs(html string) string { return htmlLayoutAttrPattern.ReplaceAllString(html, "") } + +// splitHTMLTableRows puts each table row on its own line. The chunker splits +// on "\n", so a single-line HTML table would otherwise be unsplittable and get +// force-cut at the absolute max size. Whitespace between tags is insignificant +// in HTML, and newlines are only inserted before (never inside a tag). +// A that already starts on its own line is left alone. +func splitHTMLTableRows(block string) string { + locs := htmlTableRowPattern.FindAllStringIndex(block, -1) + if len(locs) == 0 { + return strings.TrimSpace(block) + } + var b strings.Builder + last := 0 + for _, loc := range locs { + b.WriteString(block[last:loc[0]]) + if loc[0] == 0 || block[loc[0]-1] != '\n' { + b.WriteByte('\n') + } + b.WriteString(block[loc[0]:loc[1]]) + last = loc[1] + } + b.WriteString(block[last:]) + return strings.TrimSpace(b.String()) +} diff --git a/internal/infrastructure/docparser/html_table_normalizer_test.go b/internal/infrastructure/docparser/html_table_normalizer_test.go index 8ae31d59..19b036bb 100644 --- a/internal/infrastructure/docparser/html_table_normalizer_test.go +++ b/internal/infrastructure/docparser/html_table_normalizer_test.go @@ -16,7 +16,7 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) { 结尾。` - got := normalizeHTMLTables(input) + got := NormalizeHTMLTables(input) if strings.Contains(got, "` + + `` + + `` + + `` + + `
指标数值
营收10亿
` + + got := NormalizeHTMLTables(input) + + if strings.Contains(got, "1 / rowspan>1) cannot be GFM, so the table stays + // HTML — but each row must start on its own line so the chunker can split it. + input := `` + + `` + + `
合计
A1
B2
` + + got := NormalizeHTMLTables(input) + + if !strings.Contains(got, " to start on a new line, got:\n%q", got) + } + if !strings.Contains(got, "\n\n\n\n") { + t.Fatalf("expected block to be padded at the tail with blank lines, got:\n%q", got) + } + if n := strings.Count(got, "\n at offset %d is not preceded by a newline:\n%s", abs, got) + } + i = abs + 1 + } +} + func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) { // rowspan/colspan cannot be expressed in Markdown, so the table stays HTML - // but its presentational attributes are stripped. - input := `` + + // but its presentational attributes are stripped and rows become splittable. + input := `
合计
` + + `` + `
合计
AB
` - got := normalizeHTMLTables(input) + got := NormalizeHTMLTables(input) if !strings.Contains(got, " inside a cell makes the table plugin give up (it emits a newline). + // The HTML fallback must still put each on its own line. + input := `` + + `
指标
名称
数值
单位
拉伸强度
MPa
合格
A级
` + + got := NormalizeHTMLTables(input) + + if !strings.Contains(got, " table to remain HTML rather than flattened text, got:\n%s", got) + } + assertHTMLRowsSplittable(t, got) + for _, want := range []string{"指标", "名称", "拉伸强度", "合格"} { + if !strings.Contains(got, want) { + t.Fatalf("expected fallback HTML to retain %q, got:\n%s", want, got) + } + } +} + +func TestNormalizeHTMLTables_ListInCellFallsBackToSplittableHTML(t *testing.T) { + input := `` + + `
  • a
  • b
x
cd
` + + got := NormalizeHTMLTables(input) + + if !strings.Contains(got, "目录第一章第二章` + + got := NormalizeHTMLTables(input) + + if strings.Contains(got, "profile` + + got := NormalizeHTMLTables(input) + + if strings.Contains(got, "![cover](local://images/cover.png)说明` + + got := NormalizeHTMLTables(input) + + if strings.Contains(got, `!\[`) { + t.Fatalf("markdown image syntax escaped:\n%s", got) + } + if !strings.Contains(got, "![cover](local://images/cover.png)") { + t.Fatalf("expected markdown image to survive conversion, got:\n%s", got) + } +} + +func TestNormalizeHTMLTables_CodeFenceTableLeftAlone(t *testing.T) { + input := "示例:\n\n```html\n
ab
\n```\n" + + got := NormalizeHTMLTables(input) + + if got != input { + t.Fatalf("expected fenced HTML table to be unchanged, got:\n%s", got) + } +} + +func TestNormalizeHTMLTables_SpanTableIdempotent(t *testing.T) { + input := `
汇总
备注
` + once := NormalizeHTMLTables(input) + twice := NormalizeHTMLTables(once) + if once != twice { + t.Fatalf("NormalizeHTMLTables is not idempotent for span tables:\n1=%q\n2=%q", once, twice) } } diff --git a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go index 98cfc013..c05c9561 100644 --- a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go +++ b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go @@ -78,7 +78,7 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques return nil, fmt.Errorf("PaddleOCR-VL Cloud fetch results: %w", err) } - mdContent = normalizeHTMLTables(mdContent) + mdContent = NormalizeHTMLTables(mdContent) imageRefs := c.downloadImages(mdContent, imagesURL) mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs) diff --git a/internal/infrastructure/docparser/paddleocr_vl_converter.go b/internal/infrastructure/docparser/paddleocr_vl_converter.go index 313d3b74..f49e1df5 100644 --- a/internal/infrastructure/docparser/paddleocr_vl_converter.go +++ b/internal/infrastructure/docparser/paddleocr_vl_converter.go @@ -65,7 +65,7 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (* // wastes tokens and defeats the chunker's table-protection logic. Convert // them to Markdown tables (or strip layout attributes when conversion is // not possible) before downstream processing. - mdContent = normalizeHTMLTables(mdContent) + mdContent = NormalizeHTMLTables(mdContent) imageRefs, mdContent := c.processImages(mdContent, imagesB64) mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)