fix(chunk-merge): stop dropping content when contiguous chunks repeat text
AppendWithOverlap deduplicates chunker-prepended table headers by
matching a suffix of acc against a head window of next. When chunks are
strictly contiguous (positionOverlap == 0) there is no overlap to
deduplicate, but the loop still ran. If a sentence at the tail of acc
reappeared as real content inside next's head window (same sentence
written multiple times in the document), the head of next was mistaken
for a prepended header and deleted — silently losing up to hundreds of
bytes from the merged result.
This affects three callers:
- chat_pipeline.mergeOverlappingChunks: RAG search results returned
to the LLM with truncated content
- wiki_ingest.reconstructContent: distillation text fed to the LLM
is truncated, so wiki search returns 0 hits for present content
- frontend doc-content.vue port: the "merged" document view drops
the same segment
Fix: skip text matching when positionOverlap <= 0 and concatenate
verbatim. The chunker's prepended-header case only occurs inside the
overlap region (positionOverlap > 0), which still runs the loop.
Add a regression test covering a compliance-doc scenario where a
logging-requirement sentence recurs in a later Compliance section.
main
parent
abe96981c7
commit
2a60a7f30a
|
|
@ -527,11 +527,17 @@ const viewMode = ref<'chunks' | 'merged' | 'preview'>('merged');
|
|||
* 2. HTML 实体编码(" 等)导致的 content 长度与原文区间不一致——比对的是
|
||||
* 文本本身,不受长度偏差影响。
|
||||
*
|
||||
* positionOverlap <= 0 时两段位置上严格相邻或不相交,无可去重重叠;此时进入
|
||||
* 文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中 acc 后缀的真实
|
||||
* 内容重复(如同一句话在文档多次出现),把 next 开头整段误判为补写表头删掉,
|
||||
* 造成不可逆的内容丢失。直接拼接,补写表头重复交给调用方后处理。
|
||||
*
|
||||
* @param positionOverlap 由 start/end 估算的重叠量,仅用于界定搜索窗口大小。
|
||||
*/
|
||||
const appendChunkContent = (acc: string, next: string, positionOverlap: number): string => {
|
||||
if (!acc) return next;
|
||||
if (!next) return acc;
|
||||
if (positionOverlap <= 0) return acc + next;
|
||||
|
||||
const MIN_OVERLAP = 12; // 过短的后缀容易误匹配(如分隔行),忽略
|
||||
const span = Math.max(positionOverlap, 0);
|
||||
|
|
|
|||
|
|
@ -98,6 +98,12 @@ const (
|
|||
// positionOverlap 是由 StartAt/EndAt 估算的重叠量(lastEnd - curStart),仅用于
|
||||
// 界定搜索窗口大小;真正的重叠按文本匹配,能兼容补写表头与 HTML 实体长度偏差。
|
||||
// 若找不到文本重叠,则原样拼接(不裁剪),宁可保留也不破坏内容。
|
||||
//
|
||||
// positionOverlap <= 0 时两段位置上严格相邻或不相交,没有可去重的重叠;
|
||||
// 此时进入文本匹配会因 headSlack 下限 320 在 next 开头窗口内误命中
|
||||
// acc 后缀的真实内容重复(如同一句话在文档多次出现),把 next 开头
|
||||
// 整段误判为补写表头删掉,造成不可逆的内容丢失。直接拼接,补写表头
|
||||
// 重复交给调用方后处理。
|
||||
func AppendWithOverlap(acc, next string, positionOverlap int) string {
|
||||
if acc == "" {
|
||||
return next
|
||||
|
|
@ -105,14 +111,14 @@ func AppendWithOverlap(acc, next string, positionOverlap int) string {
|
|||
if next == "" {
|
||||
return acc
|
||||
}
|
||||
if positionOverlap <= 0 {
|
||||
return acc + next
|
||||
}
|
||||
|
||||
accRunes := []rune(acc)
|
||||
nextRunes := []rune(next)
|
||||
|
||||
span := positionOverlap
|
||||
if span < 0 {
|
||||
span = 0
|
||||
}
|
||||
|
||||
maxK := minInt(len(accRunes), len(nextRunes))
|
||||
if cap := maxInt(span*3, defaultSearchSpan); maxK > cap {
|
||||
|
|
|
|||
|
|
@ -159,3 +159,23 @@ func TestMergeTextChunks_GapSeparator(t *testing.T) {
|
|||
t.Fatalf("gap separator mismatch:\n got=%q\nwant=%q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAppendWithOverlap_ContiguousRealContentRepeat is a regression test:
|
||||
// two chunks are strictly contiguous (positionOverlap==0), but a boilerplate
|
||||
// sentence at the tail of acc reappears inside the head window of next as
|
||||
// real content (the same sentence is written multiple times in the document).
|
||||
// The old algorithm would enter text matching and, due to the headSlack
|
||||
// floor of 320, mistake the head of next for a prepended table header and
|
||||
// delete it, causing irreversible content loss. After the fix it should
|
||||
// concatenate directly without trimming.
|
||||
func TestAppendWithOverlap_ContiguousRealContentRepeat(t *testing.T) {
|
||||
repeat := "The system shall maintain a complete audit trail of all transactions."
|
||||
acc := "3.2 Logging Requirements\n\n" + repeat
|
||||
next := "\n\n5.1 Security Controls\n\n* Role-based access\n* Encryption at rest\n\n5.2 Compliance\n\n" + repeat + " This satisfies SOC 2."
|
||||
got := AppendWithOverlap(acc, next, 0)
|
||||
want := acc + next
|
||||
if got != want {
|
||||
t.Fatalf("contiguous real-content repeat must not trim:\n got.len=%d\nwant.len=%d\n got.tail=%q\nwant.tail=%q",
|
||||
len(got), len(want), got[len(acc)-50:], want[len(acc)-50:])
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue