diff --git a/internal/application/service/chat_pipeline/merge_position_path_test.go b/internal/application/service/chat_pipeline/merge_position_path_test.go index 8595cf0b..d7b3f034 100644 --- a/internal/application/service/chat_pipeline/merge_position_path_test.go +++ b/internal/application/service/chat_pipeline/merge_position_path_test.go @@ -55,6 +55,52 @@ func TestGroupAndMerge_TrustedOverlapTrimsAndExtendsRange(t *testing.T) { assert.ElementsMatch(t, []string{"c2"}, results[0].SubChunkID) } +func TestGroupAndMerge_TrustedOverlapBeyondTextSearchWindow(t *testing.T) { + // A real overlap wider than the text matcher's 400-rune window: the position + // path knows the exact amount, so the overlap must not survive twice. + overlap := rangeText('o', 500) + first := trustedResult("c1", 1, 0, 1000, rangeText('a', 500)+overlap) + second := trustedResult("c2", 2, 500, 1500, overlap+rangeText('b', 500)) + + results := mergeGrouped(docChunks(first, second)) + + require.Len(t, results, 1) + assert.Equal(t, rangeText('a', 500)+overlap+rangeText('b', 500), results[0].Content) + assert.Equal(t, 1500, results[0].EndAt) +} + +func TestGroupAndMerge_TrustedAdjacentRepeatedTextKeepsEveryRow(t *testing.T) { + // Adjacent trusted chunks (positional overlap 0) made of one repeating row: + // searching for the longest suffix match would mistake the repetition for an + // overlap and drop rows, so the exact-overlap path must concatenate verbatim. + row := "| cell | cell |\n" + firstBody := rangeText('x', 100) + strings.Repeat(row, 2) + secondBody := strings.Repeat(row, 3) + rangeText('y', 60) + first := trustedResult("c1", 1, 0, runeLen(firstBody), firstBody) + second := trustedResult("c2", 2, runeLen(firstBody), runeLen(firstBody)+runeLen(secondBody), secondBody) + + results := mergeGrouped(docChunks(first, second)) + + require.Len(t, results, 1) + assert.Equal(t, firstBody+secondBody, results[0].Content) + assert.Equal(t, 5, strings.Count(results[0].Content, row), "no repeated row may be dropped") +} + +func TestGroupAndMerge_TrustedOverlapTextMismatchFallsBackToTextMatch(t *testing.T) { + // Length invariant holds but the bodies disagree inside the overlap window + // (HTML entities, synthetic headers): the exact path must refuse and the text + // fallback must keep both bodies rather than trim on coordinates. + first := trustedResult("c1", 1, 0, 100, rangeText('a', 80)+rangeText('b', 20)) + second := trustedResult("c2", 2, 80, 200, rangeText('c', 20)+rangeText('d', 100)) + + results := mergeGrouped(docChunks(first, second)) + + require.Len(t, results, 1) + assert.Contains(t, results[0].Content, rangeText('b', 20)) + assert.Contains(t, results[0].Content, rangeText('c', 20)) + assert.Contains(t, results[0].Content, rangeText('d', 100)) +} + func TestGroupAndMerge_TrustedAdjacentJoinsSeamlessly(t *testing.T) { // Adjacent trusted chunks are contiguous in the original document: the // merged body must be the exact concatenation, with no separator inserted. diff --git a/internal/searchutil/chunkmerge_test.go b/internal/searchutil/chunkmerge_test.go index 5ce2bfff..e8982bb8 100644 --- a/internal/searchutil/chunkmerge_test.go +++ b/internal/searchutil/chunkmerge_test.go @@ -58,6 +58,50 @@ func TestAppendWithOverlap_NoOverlap(t *testing.T) { } } +func TestAppendWithExactOverlap_TrimsKnownOverlap(t *testing.T) { + overlap := "shared boundary text" + got, ok := AppendWithExactOverlap("before "+overlap, overlap+" after", len([]rune(overlap))) + if !ok { + t.Fatal("exact overlap should be accepted") + } + want := "before " + overlap + " after" + if got != want { + t.Fatalf("exact overlap mismatch:\n got=%q\nwant=%q", got, want) + } +} + +func TestAppendWithExactOverlap_ZeroOverlapConcatenatesRepeatedText(t *testing.T) { + // 首尾相接的两段都由同一周期性文本组成(表格行 / 日志行)。重叠量为 0 时 + // 必须原样拼接,不能去搜索后缀匹配,否则会吃掉 next 开头的重复行。 + row := "| cell | cell |\n" + acc := "前言\n" + row + row + next := row + row + row + "结尾\n" + got, ok := AppendWithExactOverlap(acc, next, 0) + if !ok { + t.Fatal("zero overlap should be accepted") + } + if got != acc+next { + t.Fatalf("zero overlap must concatenate verbatim:\n got=%q\nwant=%q", got, acc+next) + } +} + +func TestAppendWithExactOverlap_RejectsMismatchedOverlap(t *testing.T) { + // 长度不变式成立但文本已经不同(HTML 实体、补写表头等),必须拒绝, + // 由调用方回退到按文本匹配。 + if _, ok := AppendWithExactOverlap("abcdefghijkl", "XYZdefghijkl", 6); ok { + t.Fatal("mismatched overlap should be rejected") + } +} + +func TestAppendWithExactOverlap_RejectsOverlapLongerThanBodies(t *testing.T) { + if _, ok := AppendWithExactOverlap("short", "shorter", 99); ok { + t.Fatal("overlap exceeding body length should be rejected") + } + if _, ok := AppendWithExactOverlap("short", "shorter", -1); ok { + t.Fatal("negative overlap should be rejected") + } +} + func TestJoinChunkContentUsesCurrentTextInsteadOfSourceOffsets(t *testing.T) { first := "first edited body with no original overlap" second := "second independently edited body"