From cbfc0fed4a1cb4a3ff0ffabe943d6f1edfa3b21e Mon Sep 17 00:00:00 2001 From: linus-liu-web <395141741@qq.com> Date: Fri, 9 Oct 2026 10:54:37 +0800 Subject: [PATCH] =?UTF-8?q?fix(chat):=20=E5=9C=A8=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E4=B8=8A=E4=B8=8B=E6=96=87=E4=B8=AD=E6=A0=87=E6=B3=A8=20FILTER?= =?UTF-8?q?=5FTOP=5FK=20=E7=9A=84=E5=AD=90=E9=9B=86=E6=88=AA=E6=96=AD?= =?UTF-8?q?=EF=BC=8C=E9=81=BF=E5=85=8D=E6=8A=8A=E5=89=8D=20N=20=E6=9D=A1?= =?UTF-8?q?=E5=BD=93=E4=BD=9C=E5=85=A8=E9=9B=86=E4=BD=9C=E7=AD=94=20(#3758?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(chat): 在模型上下文中标注 FILTER_TOP_K 的子集截断 FILTER_TOP_K 排序后直接 searchResult[:topK],模型只拿到 topK 条却按全集 语气作答(聚合类问题最明显:名单只被截取排名靠前的若干条,模型仍输出 "这些人是……"),属于完整但错误的答案。 本次只让截断对模型可见:真正丢结果时记录 RetrievalTruncation{Shown, Candidates};prepareMessagesWithModelContext 把它作为 retrieval_shown / retrieval_candidates 传给 modelcontext;annotateSearchNotes 在 内追加 ,说明这里只是排名 靠前的若干条、无法据此给出完整名单或精确计数。未截断时字段为 nil,不产 生任何标注。 提示落在 model context 视图(模型真正读到的那份)而不是 RenderedContexts:后者会被 prepareMessagesWithModelContext 整段替换成 视图,写在那里的说明到不了模型。 不做日志:截断几乎每次检索都会触发,#3565 把 info 升 Warn 已被维护者以 "只会变噪音"否决;这里要解决的是模型看不到截断,不是运维看不到。 不动检索策略:不提高 topK、不改召回与重排,避免影响既有延迟和答案质量。 测试(夹具为合成数值): - filter_top_k_test.go:截断时 Truncation={Shown:topK,Candidates:before}, 未截断(结果数 ≤ topK)时为 nil; - modelcontext/omitted_results_test.go:带 retrieval_shown=5 / retrieval_candidates=48 时输出 且仍 以 结尾;shown==candidates 或计数缺失时不出现; - references_test.go:Truncation 非 nil 时最终 model context 含 ,nil 时不含;另有端到端用例串起 FILTER_TOP_K → INTO_CHAT_MESSAGE → prepareMessagesWithModelContext,确认 提示到达模型消息而不是被替换掉。 * fix(chat): 截断提示改条件式并覆盖重排/合并的提前裁剪 评审意见(#3758):原提示无条件要求模型 "say the answer may be incomplete", 会让普通事实问答也带上免责声明;且重排阶段与 merge 回退已可能提前裁剪, FILTER_TOP_K 不再截断时这条提示根本不生成。 - subset 提示改用评审给定的条件式措辞:只有问题在计数或要求完整列举时 才说明结果范围,其他问题照常回答;N/M 填实际计数 - 明确 M 的语义:它是首次裁剪阶段(rerank / merge / filter_top_k)收到的 候选片段数,不是全库匹配总数 - 重排阶段的阈值过滤、候选上限与 MMR top-k 裁剪,以及 merge 回退到 RerankTopK 的裁剪,都经 ChatManage.RecordRetrievalCut 记录;首次裁剪为准, 后续更窄的裁剪不覆盖候选池 - references 用实际渲染的知识片段数作为 N;补重排提前裁剪、merge 提前裁剪 到模型消息的端到端回归测试 --------- Co-authored-by: linus-liu-web <11640009+linus-liu-web@users.noreply.github.com> Co-authored-by: linus-liu-web --- .../service/chat_pipeline/filter_top_k.go | 1 + .../chat_pipeline/filter_top_k_test.go | 100 +++++++++++ .../service/chat_pipeline/merge.go | 7 + .../service/chat_pipeline/references.go | 18 +- .../service/chat_pipeline/references_test.go | 157 ++++++++++++++++++ .../service/chat_pipeline/rerank.go | 14 ++ .../service/chat_pipeline/rerank_test.go | 85 ++++++++++ internal/modelcontext/model_output.go | 27 ++- internal/modelcontext/omitted_results_test.go | 40 +++++ internal/types/chat_manage.go | 51 ++++++ 10 files changed, 493 insertions(+), 7 deletions(-) diff --git a/internal/application/service/chat_pipeline/filter_top_k.go b/internal/application/service/chat_pipeline/filter_top_k.go index 6af57996..2cd400a3 100644 --- a/internal/application/service/chat_pipeline/filter_top_k.go +++ b/internal/application/service/chat_pipeline/filter_top_k.go @@ -45,6 +45,7 @@ func (p *PluginFilterTopK) OnEvent(ctx context.Context, "before": len(searchResult), "after": topK, }) + chatManage.RecordRetrievalCut(types.RetrievalStageFilterTopK, len(searchResult)) searchResult = searchResult[:topK] } return searchResult diff --git a/internal/application/service/chat_pipeline/filter_top_k_test.go b/internal/application/service/chat_pipeline/filter_top_k_test.go index c518d634..5e2b1d79 100644 --- a/internal/application/service/chat_pipeline/filter_top_k_test.go +++ b/internal/application/service/chat_pipeline/filter_top_k_test.go @@ -2,6 +2,7 @@ package chatpipeline import ( "context" + "fmt" "testing" "github.com/Tencent/WeKnora/internal/types" @@ -59,6 +60,105 @@ func TestPluginFilterTopKUsesDeterministicTieBreakers(t *testing.T) { assert.Equal(t, []string{"chunk-c", "chunk-a", "chunk-b"}, searchResultIDs(chatManage.MergeResult)) } +// Truncation is the evidence that the prompt context is a subset: it must be +// recorded exactly when results were dropped, naming the stage and the pool it +// cut. +func TestPluginFilterTopKRecordsTruncation(t *testing.T) { + chatManage := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{RerankTopK: 2}, + PipelineState: types.PipelineState{ + MergeResult: []*types.SearchResult{ + {ID: "first", KnowledgeID: "doc-a", Score: 0.9}, + {ID: "second", KnowledgeID: "doc-b", Score: 0.8}, + {ID: "third", KnowledgeID: "doc-c", Score: 0.7}, + {ID: "fourth", KnowledgeID: "doc-d", Score: 0.6}, + }, + }, + } + + plugin := &PluginFilterTopK{} + err := plugin.OnEvent( + context.Background(), + types.FILTER_TOP_K, + chatManage, + func() *PluginError { return nil }, + ) + + require.Nil(t, err) + require.Len(t, chatManage.MergeResult, 2) + require.NotNil(t, chatManage.Truncation) + assert.Equal(t, types.RetrievalTruncation{ + Stage: types.RetrievalStageFilterTopK, + Candidates: 4, + }, *chatManage.Truncation) +} + +// An earlier, wider cut is the one the prompt must report: a later +// FILTER_TOP_K cut only removes passages from the pool the model was already +// told about, and restating the pool as the narrower number would understate +// what retrieval produced. +func TestPluginFilterTopKKeepsEarlierTruncationPool(t *testing.T) { + chatManage := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{RerankTopK: 2}, + PipelineState: types.PipelineState{ + Truncation: &types.RetrievalTruncation{ + Stage: types.RetrievalStageRerank, + Candidates: 40, + }, + MergeResult: []*types.SearchResult{ + {ID: "first", KnowledgeID: "doc-a", Score: 0.9}, + {ID: "second", KnowledgeID: "doc-b", Score: 0.8}, + {ID: "third", KnowledgeID: "doc-c", Score: 0.7}, + }, + }, + } + + plugin := &PluginFilterTopK{} + err := plugin.OnEvent( + context.Background(), + types.FILTER_TOP_K, + chatManage, + func() *PluginError { return nil }, + ) + + require.Nil(t, err) + require.Len(t, chatManage.MergeResult, 2) + assert.Equal(t, types.RetrievalTruncation{ + Stage: types.RetrievalStageRerank, + Candidates: 40, + }, *chatManage.Truncation) +} + +// A prompt that saw every candidate must not carry the subset caveat, so the +// field stays nil when the result count is at or below topK. +func TestPluginFilterTopKLeavesTruncationNilWhenNothingDropped(t *testing.T) { + for _, topK := range []int{2, 5} { + t.Run(fmt.Sprintf("topK=%d", topK), func(t *testing.T) { + chatManage := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{RerankTopK: topK}, + PipelineState: types.PipelineState{ + MergeResult: []*types.SearchResult{ + {ID: "first", KnowledgeID: "doc-a", Score: 0.9}, + {ID: "second", KnowledgeID: "doc-b", Score: 0.8}, + }, + }, + } + + plugin := &PluginFilterTopK{} + err := plugin.OnEvent( + context.Background(), + types.FILTER_TOP_K, + chatManage, + func() *PluginError { return nil }, + ) + + require.Nil(t, err) + require.Len(t, chatManage.MergeResult, 2) + assert.Nil(t, chatManage.Truncation) + }) + } +} + func searchResultIDs(results []*types.SearchResult) []string { ids := make([]string, 0, len(results)) for _, result := range results { diff --git a/internal/application/service/chat_pipeline/merge.go b/internal/application/service/chat_pipeline/merge.go index 02cef553..b4b8be56 100644 --- a/internal/application/service/chat_pipeline/merge.go +++ b/internal/application/service/chat_pipeline/merge.go @@ -120,6 +120,13 @@ func (p *PluginMerge) selectInputResults(ctx context.Context, chatManage *types. kept = append(kept, r) } } + // This cut already hides candidates from the prompt, so record it + // here: with rerank skipped there is nothing left for FILTER_TOP_K to + // cut, and without the record the model would read a subset as the + // complete result set. + if len(kept) < len(result) { + chatManage.RecordRetrievalCut(types.RetrievalStageMerge, len(result)) + } result = kept } pipelineWarn(ctx, "Merge", "fallback", map[string]interface{}{ diff --git a/internal/application/service/chat_pipeline/references.go b/internal/application/service/chat_pipeline/references.go index aa69336c..b97afe91 100644 --- a/internal/application/service/chat_pipeline/references.go +++ b/internal/application/service/chat_pipeline/references.go @@ -61,12 +61,22 @@ func prepareMessagesWithModelContext( registry.RegisterSearchResults(knowledgeResults) var contextParts []string if len(knowledgeRows) > 0 { + data := map[string]interface{}{ + "display_type": "search_results", + "results": knowledgeRows, + } + // The ranked filter chain (rerank → merge → FILTER_TOP_K) may have + // dropped candidates before this view was built. Carry the recorded + // pool and the number of knowledge passages actually rendered, so the + // model context can say the passages are a subset without restating + // the pool as a knowledge-base match count. + if truncation := chatManage.Truncation; truncation != nil { + data["retrieval_shown"] = len(knowledgeRows) + data["retrieval_candidates"] = truncation.Candidates + } contextParts = append(contextParts, registry.ModelToolResult(&types.ToolResult{ Success: true, - Data: map[string]interface{}{ - "display_type": "search_results", - "results": knowledgeRows, - }, + Data: data, })) } if len(webRows) > 0 { diff --git a/internal/application/service/chat_pipeline/references_test.go b/internal/application/service/chat_pipeline/references_test.go index b590a175..0376f113 100644 --- a/internal/application/service/chat_pipeline/references_test.go +++ b/internal/application/service/chat_pipeline/references_test.go @@ -137,3 +137,160 @@ func TestPrepareMessagesWithModelContextSuppressesCitationsWhenDisabled(t *testi require.NotContains(t, messages[1].Content, "chunk-1") require.Equal(t, "answer ", refs.DecodeOutputText(`answer `)) } + +// A truncated ranked list must reach the model context view: this is the text +// the model actually reads, and RenderedContexts is replaced wholesale. N is +// the number of passages this view renders, M the candidate pool the first +// cutting stage received. +func TestPrepareMessagesWithModelContextReportsTruncatedRetrieval(t *testing.T) { + manage := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{ + Query: "who holds the CCSK certificate", + SummaryConfig: types.SummaryConfig{Prompt: "system"}, + }, + PipelineState: types.PipelineState{ + UserContent: "question", + MergeResult: []*types.SearchResult{{ + ID: "chunk-1", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1", + KnowledgeTitle: "Doc", Content: "evidence", + }}, + Truncation: &types.RetrievalTruncation{ + Stage: types.RetrievalStageRerank, + Candidates: 146, + }, + }, + } + + messages, _ := prepareMessagesWithModelContext(context.Background(), manage) + require.Contains(t, messages[1].Content, ``) + require.Contains(t, messages[1].Content, "state that the provided context contains only the top 1 of 146") + require.Contains(t, messages[1].Content, "otherwise answer normally without adding a disclaimer") + + manage.Truncation = nil + messages, _ = prepareMessagesWithModelContext(context.Background(), manage) + require.NotContains(t, messages[1].Content, "`) + require.Contains(t, messages[1].Content, "second") + require.NotContains(t, messages[1].Content, "third") +} + +// End to end on the rerank path: with a rerank model the stage that cuts is +// rerank, not FILTER_TOP_K, which finds the already cut list and does nothing. +// Without the rerank record this turn reaches the prompt with no caveat at +// all, and a count question answers from the two shown passages as if they +// were every match. +func TestRerankTruncationReachesModelMessages(t *testing.T) { + manage := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{ + Query: "how many certificates do we hold", + RerankModelID: "rr-1", + RerankTopK: 2, + RerankThreshold: 0.3, + SummaryConfig: types.SummaryConfig{Prompt: "system", ContextTemplate: "{{query}}\n{{contexts}}"}, + }, + PipelineState: types.PipelineState{ + RewriteQuery: "how many certificates do we hold", + SearchResult: []*types.SearchResult{ + {ID: "c1", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1", Content: "first", Score: 0.5}, + {ID: "c2", KnowledgeID: "doc-2", KnowledgeBaseID: "kb-1", Content: "second", Score: 0.4}, + {ID: "c3", KnowledgeID: "doc-3", KnowledgeBaseID: "kb-1", Content: "third", Score: 0.3}, + {ID: "c4", KnowledgeID: "doc-4", KnowledgeBaseID: "kb-1", Content: "fourth", Score: 0.2}, + }, + }, + } + next := func() *PluginError { return nil } + rerank := &PluginRerank{modelService: &rerankOnlyModelService{ + reranker: &fixedReranker{scores: []float64{0.9, 0.8, 0.7, 0.6}}, + }} + if err := rerank.OnEvent(context.Background(), types.CHUNK_RERANK, manage, next); err != nil { + t.Fatalf("chunk_rerank: %v", err) + } + merge := &PluginMerge{} + if err := merge.OnEvent(context.Background(), types.CHUNK_MERGE, manage, next); err != nil { + t.Fatalf("chunk_merge: %v", err) + } + filter := &PluginFilterTopK{} + if err := filter.OnEvent(context.Background(), types.FILTER_TOP_K, manage, next); err != nil { + t.Fatalf("filter_top_k: %v", err) + } + require.Len(t, manage.MergeResult, 2) + require.NotNil(t, manage.Truncation, "rerank dropped 2 of 4 candidates and must record it") + require.Equal(t, types.RetrievalTruncation{ + Stage: types.RetrievalStageRerank, + Candidates: 4, + }, *manage.Truncation) + + messages, _ := prepareMessagesWithModelContext(context.Background(), manage) + require.Contains(t, messages[1].Content, ``) +} + +// End to end on the merge fallback path: rerank produced nothing, merge cut +// the retrieval list to RerankTopK, and FILTER_TOP_K has nothing left to cut. +// The prompt must still carry the record of the earlier cut. +func TestMergeFallbackTruncationReachesModelMessages(t *testing.T) { + manage := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{ + Query: "how many certificates do we hold", + RerankTopK: 2, + SummaryConfig: types.SummaryConfig{Prompt: "system", ContextTemplate: "{{query}}\n{{contexts}}"}, + }, + PipelineState: types.PipelineState{ + SearchResult: []*types.SearchResult{ + {ID: "c1", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1", Content: "first", Score: 0.9}, + {ID: "c2", KnowledgeID: "doc-2", KnowledgeBaseID: "kb-1", Content: "second", Score: 0.8}, + {ID: "c3", KnowledgeID: "doc-3", KnowledgeBaseID: "kb-1", Content: "third", Score: 0.7}, + }, + }, + } + next := func() *PluginError { return nil } + merge := &PluginMerge{} + if err := merge.OnEvent(context.Background(), types.CHUNK_MERGE, manage, next); err != nil { + t.Fatalf("chunk_merge: %v", err) + } + filter := &PluginFilterTopK{} + if err := filter.OnEvent(context.Background(), types.FILTER_TOP_K, manage, next); err != nil { + t.Fatalf("filter_top_k: %v", err) + } + require.Len(t, manage.MergeResult, 2) + require.NotNil(t, manage.Truncation, "merge dropped 1 of 3 candidates and must record it") + require.Equal(t, types.RetrievalTruncation{ + Stage: types.RetrievalStageMerge, + Candidates: 3, + }, *manage.Truncation) + + messages, _ := prepareMessagesWithModelContext(context.Background(), manage) + require.Contains(t, messages[1].Content, ``) +} diff --git a/internal/application/service/chat_pipeline/rerank.go b/internal/application/service/chat_pipeline/rerank.go index 7d5e048f..e460fcb1 100644 --- a/internal/application/service/chat_pipeline/rerank.go +++ b/internal/application/service/chat_pipeline/rerank.go @@ -148,6 +148,20 @@ func (p *PluginRerank) OnEvent(ctx context.Context, }) return ErrSearchNothing } + // Rerank drops candidates before FILTER_TOP_K ever runs: threshold + // rejection, the MaxCandidates cap and the MMR top-k all hide passage + // candidates that retrieval produced. Record the cut from the pool the + // stage received so the prompt can say its passages are a subset, whether + // or not FILTER_TOP_K finds anything left to cut. + if dropped := len(chatManage.SearchResult) - len(chatManage.RerankResult); dropped > 0 { + chatManage.RecordRetrievalCut(types.RetrievalStageRerank, len(chatManage.SearchResult)) + pipelineInfo(ctx, "Rerank", "truncation", map[string]interface{}{ + "stage": types.RetrievalStageRerank, + "candidates": len(chatManage.SearchResult), + "kept": len(chatManage.RerankResult), + "dropped": dropped, + }) + } pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{ "filtered_cnt": len(chatManage.RerankResult), "outcome": diag.Outcome, diff --git a/internal/application/service/chat_pipeline/rerank_test.go b/internal/application/service/chat_pipeline/rerank_test.go index 628064f4..78f4c0c6 100644 --- a/internal/application/service/chat_pipeline/rerank_test.go +++ b/internal/application/service/chat_pipeline/rerank_test.go @@ -224,3 +224,88 @@ func TestMergeFallbackKeepsRerankTopK(t *testing.T) { t.Fatalf("fallback input = %+v", got) } } + +// Rerank drops candidates before FILTER_TOP_K ever runs — threshold +// rejection, the candidate cap and the MMR top-k all hide passages. The cut +// has to be recorded here, or a turn with a rerank model reaches the prompt +// with no record that its context is a subset, and a count question reads the +// shown passages as the whole result set. +func TestPluginRerankRecordsEarlyTruncation(t *testing.T) { + cm := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{RerankModelID: "rr-1", RerankTopK: 2, RerankThreshold: 0.3}, + PipelineState: types.PipelineState{ + RewriteQuery: "q", + SearchResult: []*types.SearchResult{ + {ID: "c1", Content: "first", Score: 0.5}, + {ID: "c2", Content: "second", Score: 0.4}, + {ID: "c3", Content: "third", Score: 0.3}, + {ID: "c4", Content: "fourth", Score: 0.2}, + }, + }, + } + plugin := &PluginRerank{modelService: &rerankOnlyModelService{ + reranker: &fixedReranker{scores: []float64{0.9, 0.8, 0.7, 0.6}}, + }} + + err := plugin.OnEvent(context.Background(), types.CHUNK_RERANK, cm, func() *PluginError { return nil }) + if err != nil { + t.Fatalf("OnEvent: %v", err) + } + if len(cm.RerankResult) != 2 { + t.Fatalf("rerank result = %d, want 2", len(cm.RerankResult)) + } + want := types.RetrievalTruncation{Stage: types.RetrievalStageRerank, Candidates: 4} + if cm.Truncation == nil || *cm.Truncation != want { + t.Fatalf("truncation = %+v, want %+v", cm.Truncation, want) + } +} + +// A rerank that returned every candidate it received did not hide anything, +// so the prompt must stay free of the subset caveat. +func TestPluginRerankKeepsAllCandidatesWithoutTruncation(t *testing.T) { + cm := rerankChatManage(0.3) + plugin := &PluginRerank{modelService: &rerankOnlyModelService{ + reranker: &fixedReranker{scores: []float64{0.9, 0.8}}, + }} + + err := plugin.OnEvent(context.Background(), types.CHUNK_RERANK, cm, func() *PluginError { return nil }) + if err != nil { + t.Fatalf("OnEvent: %v", err) + } + if len(cm.RerankResult) != 2 || cm.Truncation != nil { + t.Fatalf("rerank result = %d, truncation = %+v", len(cm.RerankResult), cm.Truncation) + } +} + +// The merge fallback hides candidates too, and with rerank skipped it is the +// last stage that can say so: FILTER_TOP_K sees the already cut list and finds +// nothing left to truncate. +func TestMergeFallbackRecordsTruncation(t *testing.T) { + cm := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{RerankTopK: 2}, + PipelineState: types.PipelineState{SearchResult: []*types.SearchResult{ + {ID: "low", Score: 0.1}, {ID: "high", Score: 0.9}, {ID: "mid", Score: 0.5}, + }}, + } + (&PluginMerge{}).selectInputResults(context.Background(), cm) + want := types.RetrievalTruncation{Stage: types.RetrievalStageMerge, Candidates: 3} + if cm.Truncation == nil || *cm.Truncation != want { + t.Fatalf("truncation = %+v, want %+v", cm.Truncation, want) + } +} + +// Graph hits survive the fallback cut, so a cut that only skipped graph hits +// dropped nothing the prompt would miss and must not raise the subset caveat. +func TestMergeFallbackWithoutLossRecordsNothing(t *testing.T) { + cm := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{RerankTopK: 1}, + PipelineState: types.PipelineState{SearchResult: []*types.SearchResult{ + {ID: "high", Score: 0.9}, + {ID: "graph", Score: 0, MatchType: types.MatchTypeGraph}, + }}, + } + got := (&PluginMerge{}).selectInputResults(context.Background(), cm) + if len(got) != 2 || cm.Truncation != nil { + t.Fatalf("fallback input = %d, truncation = %+v", len(got), cm.Truncation) + } +} diff --git a/internal/modelcontext/model_output.go b/internal/modelcontext/model_output.go index 7c512a21..7de0fad6 100644 --- a/internal/modelcontext/model_output.go +++ b/internal/modelcontext/model_output.go @@ -377,12 +377,21 @@ func graphTruncationNote(data map[string]interface{}, shownRelations int) string // annotateSearchNotes tells the model what a search result does not show: // how many lower-ranked results were left out to fit the tool output budget // (so it narrows the query or lowers the limit instead of concluding nothing -// else matched), and which knowledge bases could not be searched (so a -// failure is not read as an absence of evidence). +// else matched), how much of the ranked candidate pool the rendered passages +// stand for, and which knowledge bases could not be searched (so a failure is +// not read as an absence of evidence). func annotateSearchNotes(output string, data map[string]interface{}) string { omitted := intValue(data, "omitted_for_budget") failures := stringSliceValue(data["partial_failures"]) - if (omitted <= 0 && len(failures) == 0) || !strings.HasSuffix(output, "") { + // retrieval_candidates is the candidate pool of the first ranked-filter + // stage that dropped anything (rerank, merge or FILTER_TOP_K), not the + // number of matches in the knowledge base, which retrieval never computes + // at that depth; retrieval_shown is how many knowledge passages this view + // renders out of that pool. Both count passages. + shown := intValue(data, "retrieval_shown") + candidates := intValue(data, "retrieval_candidates") + truncated := shown > 0 && candidates > shown + if (omitted <= 0 && len(failures) == 0 && !truncated) || !strings.HasSuffix(output, "") { return output } var b strings.Builder @@ -390,6 +399,18 @@ func annotateSearchNotes(output string, data map[string]interface{}) string { fmt.Fprintf(&b, " Lower-ranked results were left out to "+ "fit the output size. Narrow the query or lower limit to see them.\n", omitted) } + if truncated { + // The caveat is conditional on purpose: a truncated view is the norm + // for a ranked retrieval, so an unconditional "the answer may be + // incomplete" would add a disclaimer to ordinary factual answers that + // the subset does not affect. Only counting and exhaustive-list + // questions have to report the scope of what was read. + fmt.Fprintf(&b, " If the question asks for a count or an "+ + "exhaustive list, state that the provided context contains only the top %d of %d candidate "+ + "passages at this filtering stage, and do not treat these passage counts as the requested "+ + "total; otherwise answer normally without adding a disclaimer solely because of this "+ + "truncation.\n", shown, candidates, shown, candidates) + } for _, failure := range failures { fmt.Fprintf(&b, " %s — these knowledge bases were not searched.\n", escapeText(failure)) diff --git a/internal/modelcontext/omitted_results_test.go b/internal/modelcontext/omitted_results_test.go index c82184cd..f5f69cfc 100644 --- a/internal/modelcontext/omitted_results_test.go +++ b/internal/modelcontext/omitted_results_test.go @@ -22,6 +22,46 @@ func TestAnnotateSearchNotes(t *testing.T) { } } +// The subset note is conditional: only a counting or exhaustive-list question +// has to report the scope of the view. It must also carry the counts it was +// given, so the model reads the real shown / candidate numbers rather than a +// placeholder or a knowledge-base total. +func TestAnnotateSearchNotesReportsTruncatedRetrieval(t *testing.T) { + t.Parallel() + base := "\n" + got := annotateSearchNotes(base, map[string]interface{}{ + "retrieval_shown": 5, + "retrieval_candidates": 146, + }) + want := ` If the question asks for a count or an exhaustive list, ` + + `state that the provided context contains only the top 5 of 146 candidate passages at this ` + + `filtering stage, and do not treat these passage counts as the requested total; otherwise ` + + `answer normally without adding a disclaimer solely because of this truncation.` + if !strings.Contains(got, want) { + t.Fatalf("annotated = %q\nwant subset %q", got, want) + } + if strings.Contains(got, "say the answer may be incomplete") { + t.Fatalf("the unconditional disclaimer must be gone: %q", got) + } + if !strings.HasSuffix(got, "") { + t.Fatalf("subset note must stay inside the retrieval block: %q", got) + } + // A prompt that saw every candidate, or was given no usable counts, + // carries no caveat at all. + for name, data := range map[string]map[string]interface{}{ + "complete": {"retrieval_shown": 5, "retrieval_candidates": 5}, + "no counts": {}, + "shown only": {"retrieval_shown": 5}, + "candidates only": {"retrieval_candidates": 146}, + "more than pool": {"retrieval_shown": 7, "retrieval_candidates": 5}, + "no shown passage": {"retrieval_shown": 0, "retrieval_candidates": 146}, + } { + if annotateSearchNotes(base, data) != base { + t.Fatalf("%s: a non-truncated retrieval must leave the output unchanged", name) + } + } +} + func TestAnnotateGraphResult(t *testing.T) { t.Parallel() base := "\n" diff --git a/internal/types/chat_manage.go b/internal/types/chat_manage.go index 54b6ec14..d21d4b5e 100644 --- a/internal/types/chat_manage.go +++ b/internal/types/chat_manage.go @@ -115,6 +115,52 @@ func (i QueryIntent) NeedsKBRetrieval() bool { } } +// Retrieval stage names recorded on RetrievalTruncation.Stage, in pipeline +// order. +const ( + // RetrievalStageRerank is the CHUNK_RERANK stage, which drops candidates + // through threshold rejection, its candidate cap or its MMR top-k. + RetrievalStageRerank = "rerank" + // RetrievalStageMerge is the CHUNK_MERGE fallback cut to RerankTopK, taken + // when rerank produced nothing to merge. + RetrievalStageMerge = "merge" + // RetrievalStageFilterTopK is the FILTER_TOP_K stage. + RetrievalStageFilterTopK = "filter_top_k" +) + +// RetrievalTruncation records that the passages in the model context are a +// subset of this turn's ranked candidates. +// +// Stage is the first pipeline stage that dropped candidates and Candidates is +// the number of candidate passages that stage received, so the pair always +// describes one population: the retrieval set that entered the ranked filter +// chain (rerank → merge → FILTER_TOP_K). Candidates is a pipeline-local count +// of retrieved passages — never the number of matches in the knowledge base, +// which retrieval never computes — and a later, narrower cut never replaces +// it. How many of those candidates the prompt actually shows is counted where +// the prompt is built, so the caveat is only emitted when the context really +// is a subset. +type RetrievalTruncation struct { + Stage string `json:"stage"` + Candidates int `json:"candidates"` +} + +// RecordRetrievalCut notes that a ranked-list stage dropped candidate passages +// before the model context was built. candidates is the population that stage +// received, read before the cut; it must never be a knowledge-base match count, +// which retrieval does not compute at this depth. +// +// The first cut wins. Later stages only see an equal or narrower population +// (FILTER_TOP_K cuts merge output, merge's fallback cuts the search output), so +// letting them overwrite the record would restate the candidate pool as +// something smaller than what retrieval produced and hide how much was dropped. +func (c *ChatManage) RecordRetrievalCut(stage string, candidates int) { + if c == nil || c.Truncation != nil || candidates <= 0 { + return + } + c.Truncation = &RetrievalTruncation{Stage: stage, Candidates: candidates} +} + // PipelineState holds mutable intermediate data that plugins read and write // as the pipeline progresses. type PipelineState struct { @@ -138,6 +184,11 @@ type PipelineState struct { ImageDescription string `json:"-"` QuotedContext string `json:"-"` // Quoted message text, injected at LLM prompt stage SystemPromptOverride string `json:"-"` + // Truncation records the first ranked-list cut this turn: the stage that + // dropped candidates and the size of the candidate pool it cut. Nil when + // no stage dropped anything, so a prompt only carries the caveat when its + // context really is a subset. + Truncation *RetrievalTruncation `json:"-"` // MemoryPrompt is the long-term memory envelope appended to the system // prompt for this turn, empty when memory is off or nothing matched. MemoryPrompt string `json:"-"`