fix(chunk-merge): stop dropping content when contiguous chunks repeat text

AppendWithOverlap deduplicates chunker-prepended table headers by
  matching a suffix of acc against a head window of next. When chunks are
  strictly contiguous (positionOverlap == 0) there is no overlap to
  deduplicate, but the loop still ran. If a sentence at the tail of acc
  reappeared as real content inside next's head window (same sentence
  written multiple times in the document), the head of next was mistaken
  for a prepended header and deleted — silently losing up to hundreds of
  bytes from the merged result.

  This affects three callers:
    - chat_pipeline.mergeOverlappingChunks: RAG search results returned
      to the LLM with truncated content
    - wiki_ingest.reconstructContent: distillation text fed to the LLM
      is truncated, so wiki search returns 0 hits for present content
    - frontend doc-content.vue port: the "merged" document view drops
      the same segment

  Fix: skip text matching when positionOverlap <= 0 and concatenate
  verbatim. The chunker's prepended-header case only occurs inside the
  overlap region (positionOverlap > 0), which still runs the loop.

  Add a regression test covering a compliance-doc scenario where a
  logging-requirement sentence recurs in a later Compliance section.
main
ChenRussell 2026-08-13 14:28:24 +08:00 committed by lyingbug
parent abe96981c7
commit 2a60a7f30a
3 changed files with 35 additions and 3 deletions

View File

@ -527,11 +527,17 @@ const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged');
* 2. HTML 实体编码(&#34; 等)导致的 content 长度与原文区间不一致——比对的是
* 文本本身,不受长度偏差影响。
*
* positionOverlap <= 0 时两段位置上严格相邻或不相交,无可去重重叠;此时进入
* 文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中 acc 后缀的真实
* 内容重复(如同一句话在文档多次出现),把 next 开头整段误判为补写表头删掉,
* 造成不可逆的内容丢失。直接拼接,补写表头重复交给调用方后处理。
*
* @param positionOverlap 由 start/end 估算的重叠量,仅用于界定搜索窗口大小。
*/
const appendChunkContent = (acc: string, next: string, positionOverlap: number): string => {
if (!acc) return next;
if (!next) return acc;
if (positionOverlap <= 0) return acc + next;
const MIN_OVERLAP = 12; // 过短的后缀容易误匹配(如分隔行),忽略
const span = Math.max(positionOverlap, 0);

View File

@ -98,6 +98,12 @@ const (
// positionOverlap 是由 StartAt/EndAt 估算的重叠量(lastEnd - curStart),仅用于
// 界定搜索窗口大小;真正的重叠按文本匹配,能兼容补写表头与 HTML 实体长度偏差。
// 若找不到文本重叠,则原样拼接(不裁剪),宁可保留也不破坏内容。
//
// positionOverlap <= 0 时两段位置上严格相邻或不相交,没有可去重的重叠;
// 此时进入文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中
// acc 后缀的真实内容重复(如同一句话在文档多次出现),把 next 开头
// 整段误判为补写表头删掉,造成不可逆的内容丢失。直接拼接,补写表头
// 重复交给调用方后处理。
func AppendWithOverlap(acc, next string, positionOverlap int) string {
if acc == "" {
return next
@ -105,14 +111,14 @@ func AppendWithOverlap(acc, next string, positionOverlap int) string {
if next == "" {
return acc
}
if positionOverlap <= 0 {
return acc + next
}
accRunes := []rune(acc)
nextRunes := []rune(next)
span := positionOverlap
if span < 0 {
span = 0
}
maxK := minInt(len(accRunes), len(nextRunes))
if cap := maxInt(span*3, defaultSearchSpan); maxK > cap {

View File

@ -159,3 +159,23 @@ func TestMergeTextChunks_GapSeparator(t *testing.T) {
t.Fatalf("gap separator mismatch:\n got=%q\nwant=%q", got, want)
}
}
// TestAppendWithOverlap_ContiguousRealContentRepeat is a regression test:
// two chunks are strictly contiguous (positionOverlap==0), but a boilerplate
// sentence at the tail of acc reappears inside the head window of next as
// real content (the same sentence is written multiple times in the document).
// The old algorithm would enter text matching and, due to the headSlack
// floor of 320, mistake the head of next for a prepended table header and
// delete it, causing irreversible content loss. After the fix it should
// concatenate directly without trimming.
func TestAppendWithOverlap_ContiguousRealContentRepeat(t *testing.T) {
repeat := "The system shall maintain a complete audit trail of all transactions."
acc := "3.2 Logging Requirements\n\n" + repeat
next := "\n\n5.1 Security Controls\n\n* Role-based access\n* Encryption at rest\n\n5.2 Compliance\n\n" + repeat + " This satisfies SOC 2."
got := AppendWithOverlap(acc, next, 0)
want := acc + next
if got != want {
t.Fatalf("contiguous real-content repeat must not trim:\n got.len=%d\nwant.len=%d\n got.tail=%q\nwant.tail=%q",
len(got), len(want), got[len(acc)-50:], want[len(acc)-50:])
}
}