From 2a60a7f30aa7414f4fcebe779218ee4e60b99248 Mon Sep 17 00:00:00 2001 From: ChenRussell <792608290@qq.com> Date: Thu, 13 Aug 2026 14:28:24 +0800 Subject: [PATCH] fix(chunk-merge): stop dropping content when contiguous chunks repeat text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AppendWithOverlap deduplicates chunker-prepended table headers by matching a suffix of acc against a head window of next. When chunks are strictly contiguous (positionOverlap == 0) there is no overlap to deduplicate, but the loop still ran. If a sentence at the tail of acc reappeared as real content inside next's head window (same sentence written multiple times in the document), the head of next was mistaken for a prepended header and deleted — silently losing up to hundreds of bytes from the merged result. This affects three callers: - chat_pipeline.mergeOverlappingChunks: RAG search results returned to the LLM with truncated content - wiki_ingest.reconstructContent: distillation text fed to the LLM is truncated, so wiki search returns 0 hits for present content - frontend doc-content.vue port: the "merged" document view drops the same segment Fix: skip text matching when positionOverlap <= 0 and concatenate verbatim. The chunker's prepended-header case only occurs inside the overlap region (positionOverlap > 0), which still runs the loop. Add a regression test covering a compliance-doc scenario where a logging-requirement sentence recurs in a later Compliance section. --- frontend/src/components/doc-content.vue | 6 ++++++ internal/searchutil/chunkmerge.go | 12 +++++++++--- internal/searchutil/chunkmerge_test.go | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/doc-content.vue b/frontend/src/components/doc-content.vue index 3fcabdfd..614962fb 100644 --- a/frontend/src/components/doc-content.vue +++ b/frontend/src/components/doc-content.vue @@ -527,11 +527,17 @@ const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged'); * 2. HTML 实体编码(" 等)导致的 content 长度与原文区间不一致——比对的是 * 文本本身,不受长度偏差影响。 * + * positionOverlap <= 0 时两段位置上严格相邻或不相交,无可去重重叠;此时进入 + * 文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中 acc 后缀的真实 + * 内容重复(如同一句话在文档多次出现),把 next 开头整段误判为补写表头删掉, + * 造成不可逆的内容丢失。直接拼接,补写表头重复交给调用方后处理。 + * * @param positionOverlap 由 start/end 估算的重叠量,仅用于界定搜索窗口大小。 */ const appendChunkContent = (acc: string, next: string, positionOverlap: number): string => { if (!acc) return next; if (!next) return acc; + if (positionOverlap <= 0) return acc + next; const MIN_OVERLAP = 12; // 过短的后缀容易误匹配(如分隔行),忽略 const span = Math.max(positionOverlap, 0); diff --git a/internal/searchutil/chunkmerge.go b/internal/searchutil/chunkmerge.go index 7c8669b0..a8149577 100644 --- a/internal/searchutil/chunkmerge.go +++ b/internal/searchutil/chunkmerge.go @@ -98,6 +98,12 @@ const ( // positionOverlap 是由 StartAt/EndAt 估算的重叠量(lastEnd - curStart),仅用于 // 界定搜索窗口大小;真正的重叠按文本匹配,能兼容补写表头与 HTML 实体长度偏差。 // 若找不到文本重叠,则原样拼接(不裁剪),宁可保留也不破坏内容。 +// +// positionOverlap <= 0 时两段位置上严格相邻或不相交,没有可去重的重叠; +// 此时进入文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中 +// acc 后缀的真实内容重复(如同一句话在文档多次出现),把 next 开头 +// 整段误判为补写表头删掉,造成不可逆的内容丢失。直接拼接,补写表头 +// 重复交给调用方后处理。 func AppendWithOverlap(acc, next string, positionOverlap int) string { if acc == "" { return next @@ -105,14 +111,14 @@ func AppendWithOverlap(acc, next string, positionOverlap int) string { if next == "" { return acc } + if positionOverlap <= 0 { + return acc + next + } accRunes := []rune(acc) nextRunes := []rune(next) span := positionOverlap - if span < 0 { - span = 0 - } maxK := minInt(len(accRunes), len(nextRunes)) if cap := maxInt(span*3, defaultSearchSpan); maxK > cap { diff --git a/internal/searchutil/chunkmerge_test.go b/internal/searchutil/chunkmerge_test.go index e8982bb8..17de0dbe 100644 --- a/internal/searchutil/chunkmerge_test.go +++ b/internal/searchutil/chunkmerge_test.go @@ -159,3 +159,23 @@ func TestMergeTextChunks_GapSeparator(t *testing.T) { t.Fatalf("gap separator mismatch:\n got=%q\nwant=%q", got, want) } } + +// TestAppendWithOverlap_ContiguousRealContentRepeat is a regression test: +// two chunks are strictly contiguous (positionOverlap==0), but a boilerplate +// sentence at the tail of acc reappears inside the head window of next as +// real content (the same sentence is written multiple times in the document). +// The old algorithm would enter text matching and, due to the headSlack +// floor of 320, mistake the head of next for a prepended table header and +// delete it, causing irreversible content loss. After the fix it should +// concatenate directly without trimming. +func TestAppendWithOverlap_ContiguousRealContentRepeat(t *testing.T) { + repeat := "The system shall maintain a complete audit trail of all transactions." + acc := "3.2 Logging Requirements\n\n" + repeat + next := "\n\n5.1 Security Controls\n\n* Role-based access\n* Encryption at rest\n\n5.2 Compliance\n\n" + repeat + " This satisfies SOC 2." + got := AppendWithOverlap(acc, next, 0) + want := acc + next + if got != want { + t.Fatalf("contiguous real-content repeat must not trim:\n got.len=%d\nwant.len=%d\n got.tail=%q\nwant.tail=%q", + len(got), len(want), got[len(acc)-50:], want[len(acc)-50:]) + } +}