diff --git a/frontend/src/components/doc-content.vue b/frontend/src/components/doc-content.vue index 3fcabdfd..614962fb 100644 --- a/frontend/src/components/doc-content.vue +++ b/frontend/src/components/doc-content.vue @@ -527,11 +527,17 @@ const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged'); * 2. HTML 实体编码(" 等)导致的 content 长度与原文区间不一致——比对的是 * 文本本身,不受长度偏差影响。 * + * positionOverlap <= 0 时两段位置上严格相邻或不相交,无可去重重叠;此时进入 + * 文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中 acc 后缀的真实 + * 内容重复(如同一句话在文档多次出现),把 next 开头整段误判为补写表头删掉, + * 造成不可逆的内容丢失。直接拼接,补写表头重复交给调用方后处理。 + * * @param positionOverlap 由 start/end 估算的重叠量,仅用于界定搜索窗口大小。 */ const appendChunkContent = (acc: string, next: string, positionOverlap: number): string => { if (!acc) return next; if (!next) return acc; + if (positionOverlap <= 0) return acc + next; const MIN_OVERLAP = 12; // 过短的后缀容易误匹配(如分隔行),忽略 const span = Math.max(positionOverlap, 0); diff --git a/internal/searchutil/chunkmerge.go b/internal/searchutil/chunkmerge.go index 7c8669b0..a8149577 100644 --- a/internal/searchutil/chunkmerge.go +++ b/internal/searchutil/chunkmerge.go @@ -98,6 +98,12 @@ const ( // positionOverlap 是由 StartAt/EndAt 估算的重叠量(lastEnd - curStart),仅用于 // 界定搜索窗口大小;真正的重叠按文本匹配,能兼容补写表头与 HTML 实体长度偏差。 // 若找不到文本重叠,则原样拼接(不裁剪),宁可保留也不破坏内容。 +// +// positionOverlap <= 0 时两段位置上严格相邻或不相交,没有可去重的重叠; +// 此时进入文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中 +// acc 后缀的真实内容重复(如同一句话在文档多次出现),把 next 开头 +// 整段误判为补写表头删掉,造成不可逆的内容丢失。直接拼接,补写表头 +// 重复交给调用方后处理。 func AppendWithOverlap(acc, next string, positionOverlap int) string { if acc == "" { return next @@ -105,14 +111,14 @@ func AppendWithOverlap(acc, next string, positionOverlap int) string { if next == "" { return acc } + if positionOverlap <= 0 { + return acc + next + } accRunes := []rune(acc) nextRunes := []rune(next) span := positionOverlap - if span < 0 { - span = 0 - } maxK := minInt(len(accRunes), len(nextRunes)) if cap := maxInt(span*3, defaultSearchSpan); maxK > cap { diff --git a/internal/searchutil/chunkmerge_test.go b/internal/searchutil/chunkmerge_test.go index e8982bb8..17de0dbe 100644 --- a/internal/searchutil/chunkmerge_test.go +++ b/internal/searchutil/chunkmerge_test.go @@ -159,3 +159,23 @@ func TestMergeTextChunks_GapSeparator(t *testing.T) { t.Fatalf("gap separator mismatch:\n got=%q\nwant=%q", got, want) } } + +// TestAppendWithOverlap_ContiguousRealContentRepeat is a regression test: +// two chunks are strictly contiguous (positionOverlap==0), but a boilerplate +// sentence at the tail of acc reappears inside the head window of next as +// real content (the same sentence is written multiple times in the document). +// The old algorithm would enter text matching and, due to the headSlack +// floor of 320, mistake the head of next for a prepended table header and +// delete it, causing irreversible content loss. After the fix it should +// concatenate directly without trimming. +func TestAppendWithOverlap_ContiguousRealContentRepeat(t *testing.T) { + repeat := "The system shall maintain a complete audit trail of all transactions." + acc := "3.2 Logging Requirements\n\n" + repeat + next := "\n\n5.1 Security Controls\n\n* Role-based access\n* Encryption at rest\n\n5.2 Compliance\n\n" + repeat + " This satisfies SOC 2." + got := AppendWithOverlap(acc, next, 0) + want := acc + next + if got != want { + t.Fatalf("contiguous real-content repeat must not trim:\n got.len=%d\nwant.len=%d\n got.tail=%q\nwant.tail=%q", + len(got), len(want), got[len(acc)-50:], want[len(acc)-50:]) + } +}