feat(chunks): restore position-aware merge for trusted pairs

Chunk editing made parser coordinates unreliable, so all merges were
downgraded to pure text matching, including unedited documents.

Restore position-aware merging for pairs whose coordinates are still
trustworthy, and keep the text fallback for edited or stale content so
it is never dropped on coordinates.
main
MidoriKurage 2026-08-06 10:38:10 +08:00 committed by lyingbug
parent 3abeba31dd
commit b7b85621fb
1 changed files with 98 additions and 20 deletions

View File

@ -9,10 +9,11 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// mergeSequentialChunks joins sequential current chunk bodies without using
// parser StartAt/EndAt coordinates. Chunks MUST be pre-sorted by ChunkIndex.
// Source coordinates remain useful for citations, but arbitrary user edits
// make them unsafe for deciding whether current content can be discarded.
// mergeSequentialChunks joins sequential current chunk bodies.
// Trusted pairs use position-aware merging;
// any pair involving edited, expanded, or stale content falls back to
// text matching so current content is never discarded on parser coordinates.
// Chunks MUST be pre-sorted by ChunkIndex.
func (p *PluginMerge) mergeSequentialChunks(
ctx context.Context,
knowledgeID string,
@ -31,29 +32,31 @@ func (p *PluginMerge) mergeSequentialChunks(
current := chunks[i]
last := &groups[len(groups)-1]
lastChunk := last.result
textContained := searchutil.ContainsChunkContent(lastChunk.Content, current.Content) ||
searchutil.ContainsChunkContent(current.Content, lastChunk.Content)
sequential := current.ChunkIndex == last.lastIndex+1
if !textContained && !sequential {
switch classifyMerge(lastChunk, last.lastIndex, current) {
case mergeSeparate:
groups = append(groups, mergedGroup{result: current, lastIndex: current.ChunkIndex})
continue
case mergeExtend:
lastChunk.Content = searchutil.AppendWithOverlap(
lastChunk.Content, current.Content, lastChunk.EndAt-current.StartAt,
)
lastChunk.EndAt = current.EndAt
recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge")
case mergeSubsume:
recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge_contained")
case mergeJoinDistinct:
lastChunk.Content = searchutil.JoinChunkContent(lastChunk.Content, current.Content, "\n\n")
recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge_contained")
case mergeJoinText:
lastChunk.Content = searchutil.JoinChunkContent(lastChunk.Content, current.Content, "\n\n")
recordMergedChild(ctx, knowledgeID, lastChunk, current, "image_merge")
}
lastChunk.Content = searchutil.JoinChunkContent(lastChunk.Content, current.Content, "\n\n")
if !containsID(lastChunk.SubChunkID, current.ID) {
lastChunk.SubChunkID = append(lastChunk.SubChunkID, current.ID)
}
if err := mergeImageInfo(ctx, lastChunk, current); err != nil {
pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
// Extend the merged group's span and keep the higher score.
if current.ChunkIndex > last.lastIndex {
last.lastIndex = current.ChunkIndex
}
// Keep the higher score
if current.Score > lastChunk.Score {
lastChunk.Score = current.Score
}
@ -72,6 +75,81 @@ func (p *PluginMerge) mergeSequentialChunks(
return merged
}
// chunkTrusted reports whether a result's StartAt/EndAt can be trusted for
// position-based merging: unedited, valid range, and splitter-consistent
// length (runeLen(Content) == EndAt-StartAt).
func chunkTrusted(chunk *types.SearchResult) bool {
return chunk.ContentRevision == 0 &&
chunk.EndAt > chunk.StartAt &&
runeLen(chunk.Content) == chunk.EndAt-chunk.StartAt
}
// mergeSituation labels how the current result relates to the group's leading
// result in document order. The classifier decides it once; the merge loop
// switches on it without nesting.
type mergeSituation int
const (
// mergeSeparate: current is not mergeable, it starts a new group
// (position gap, or untrusted pair that is neither text-contained nor
// index-sequential).
mergeSeparate mergeSituation = iota
// mergeExtend: trusted pair with partial overlap or adjacency; the
// non-overlapping suffix is appended with text-verified trimming.
mergeExtend
// mergeSubsume: trusted pair, range-contained and text-verified; current
// is recorded without touching the merged content.
mergeSubsume
// mergeJoinDistinct: trusted pair, range-contained but textually distinct;
// current is kept via a text join, breaking the length invariant so later
// pairs degrade to the text path.
mergeJoinDistinct
// mergeJoinText: untrusted pair that is text-contained or index-sequential;
// joined via pure text matching.
mergeJoinText
)
// classifyMerge labels the pair relationship so the merge loop can dispatch
// with a flat switch. It needs the group's running lastIndex for the untrusted
// sequentiality check.
func classifyMerge(lastChunk *types.SearchResult, lastIndex int, current *types.SearchResult) mergeSituation {
if chunkTrusted(lastChunk) && chunkTrusted(current) && current.StartAt >= lastChunk.StartAt {
switch {
case current.StartAt > lastChunk.EndAt:
return mergeSeparate
case current.EndAt > lastChunk.EndAt:
return mergeExtend
case searchutil.ContainsChunkContent(lastChunk.Content, current.Content):
return mergeSubsume
default:
return mergeJoinDistinct
}
}
textContained := searchutil.ContainsChunkContent(lastChunk.Content, current.Content) ||
searchutil.ContainsChunkContent(current.Content, lastChunk.Content)
sequential := current.ChunkIndex == lastIndex+1
if !textContained && !sequential {
return mergeSeparate
}
return mergeJoinText
}
// recordMergedChild records a merged chunk on the group result: appends its ID
// to SubChunkID (deduplicated) and merges its ImageInfo, warning on failure.
// warnKey distinguishes merge contexts in pipeline diagnostics.
func recordMergedChild(ctx context.Context, knowledgeID string, target, source *types.SearchResult, warnKey string) {
if !containsID(target.SubChunkID, source.ID) {
target.SubChunkID = append(target.SubChunkID, source.ID)
}
if err := mergeImageInfo(ctx, target, source); err != nil {
pipelineWarn(ctx, "Merge", warnKey, map[string]any{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
}
// mergeImageInfo merges ImageInfo from source into target, deduplicating by URL.
func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *types.SearchResult) error {
if source.ImageInfo == "" {