fix(chat): 在模型上下文中标注 FILTER_TOP_K 的子集截断,避免把前 N 条当作全集作答 (#3758)

* fix(chat): 在模型上下文中标注 FILTER_TOP_K 的子集截断

FILTER_TOP_K 排序后直接 searchResult[:topK],模型只拿到 topK 条却按全集
语气作答(聚合类问题最明显:名单只被截取排名靠前的若干条,模型仍输出
"这些人是……"),属于完整但错误的答案。

本次只让截断对模型可见:真正丢结果时记录 RetrievalTruncation{Shown,
Candidates};prepareMessagesWithModelContext 把它作为 retrieval_shown /
retrieval_candidates 传给 modelcontext;annotateSearchNotes 在
<retrieval> 内追加 <subset shown="N" candidates="M">,说明这里只是排名
靠前的若干条、无法据此给出完整名单或精确计数。未截断时字段为 nil,不产
生任何标注。

提示落在 model context 视图(模型真正读到的那份)而不是
RenderedContexts:后者会被 prepareMessagesWithModelContext 整段替换成
<retrieval> 视图,写在那里的说明到不了模型。

不做日志:截断几乎每次检索都会触发,#3565 把 info 升 Warn 已被维护者以
"只会变噪音"否决;这里要解决的是模型看不到截断,不是运维看不到。

不动检索策略:不提高 topK、不改召回与重排,避免影响既有延迟和答案质量。

测试(夹具为合成数值):
- filter_top_k_test.go:截断时 Truncation={Shown:topK,Candidates:before},
  未截断(结果数 ≤ topK)时为 nil;
- modelcontext/omitted_results_test.go:带 retrieval_shown=5 /
  retrieval_candidates=48 时输出 <subset shown="5" candidates="48"> 且仍
  以 </retrieval> 结尾;shown==candidates 或计数缺失时不出现;
- references_test.go:Truncation 非 nil 时最终 model context 含
  <subset shown="3" candidates="48">,nil 时不含;另有端到端用例串起
  FILTER_TOP_K → INTO_CHAT_MESSAGE → prepareMessagesWithModelContext,确认
  提示到达模型消息而不是被替换掉。

* fix(chat): 截断提示改条件式并覆盖重排/合并的提前裁剪

评审意见(#3758):原提示无条件要求模型 "say the answer may be incomplete",
会让普通事实问答也带上免责声明;且重排阶段与 merge 回退已可能提前裁剪,
FILTER_TOP_K 不再截断时这条提示根本不生成。

- subset 提示改用评审给定的条件式措辞:只有问题在计数或要求完整列举时
  才说明结果范围,其他问题照常回答;N/M 填实际计数
- 明确 M 的语义:它是首次裁剪阶段(rerank / merge / filter_top_k)收到的
  候选片段数,不是全库匹配总数
- 重排阶段的阈值过滤、候选上限与 MMR top-k 裁剪,以及 merge 回退到
  RerankTopK 的裁剪,都经 ChatManage.RecordRetrievalCut 记录;首次裁剪为准,
  后续更窄的裁剪不覆盖候选池
- references 用实际渲染的知识片段数作为 N;补重排提前裁剪、merge 提前裁剪
  到模型消息的端到端回归测试

---------

Co-authored-by: linus-liu-web <11640009+linus-liu-web@users.noreply.github.com>
Co-authored-by: linus-liu-web <linus-liu-web@users.noreply.github.com>
main
linus-liu-web 2026-10-09 10:54:37 +08:00 committed by GitHub
parent 419f17fc65
commit cbfc0fed4a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 493 additions and 7 deletions

View File

@ -45,6 +45,7 @@ func (p *PluginFilterTopK) OnEvent(ctx context.Context,
"before": len(searchResult),
"after": topK,
})
chatManage.RecordRetrievalCut(types.RetrievalStageFilterTopK, len(searchResult))
searchResult = searchResult[:topK]
}
return searchResult

View File

@ -2,6 +2,7 @@ package chatpipeline
import (
"context"
"fmt"
"testing"
"github.com/Tencent/WeKnora/internal/types"
@ -59,6 +60,105 @@ func TestPluginFilterTopKUsesDeterministicTieBreakers(t *testing.T) {
assert.Equal(t, []string{"chunk-c", "chunk-a", "chunk-b"}, searchResultIDs(chatManage.MergeResult))
}
// Truncation is the evidence that the prompt context is a subset: it must be
// recorded exactly when results were dropped, naming the stage and the pool it
// cut.
func TestPluginFilterTopKRecordsTruncation(t *testing.T) {
chatManage := &types.ChatManage{
PipelineRequest: types.PipelineRequest{RerankTopK: 2},
PipelineState: types.PipelineState{
MergeResult: []*types.SearchResult{
{ID: "first", KnowledgeID: "doc-a", Score: 0.9},
{ID: "second", KnowledgeID: "doc-b", Score: 0.8},
{ID: "third", KnowledgeID: "doc-c", Score: 0.7},
{ID: "fourth", KnowledgeID: "doc-d", Score: 0.6},
},
},
}
plugin := &PluginFilterTopK{}
err := plugin.OnEvent(
context.Background(),
types.FILTER_TOP_K,
chatManage,
func() *PluginError { return nil },
)
require.Nil(t, err)
require.Len(t, chatManage.MergeResult, 2)
require.NotNil(t, chatManage.Truncation)
assert.Equal(t, types.RetrievalTruncation{
Stage: types.RetrievalStageFilterTopK,
Candidates: 4,
}, *chatManage.Truncation)
}
// An earlier, wider cut is the one the prompt must report: a later
// FILTER_TOP_K cut only removes passages from the pool the model was already
// told about, and restating the pool as the narrower number would understate
// what retrieval produced.
func TestPluginFilterTopKKeepsEarlierTruncationPool(t *testing.T) {
chatManage := &types.ChatManage{
PipelineRequest: types.PipelineRequest{RerankTopK: 2},
PipelineState: types.PipelineState{
Truncation: &types.RetrievalTruncation{
Stage: types.RetrievalStageRerank,
Candidates: 40,
},
MergeResult: []*types.SearchResult{
{ID: "first", KnowledgeID: "doc-a", Score: 0.9},
{ID: "second", KnowledgeID: "doc-b", Score: 0.8},
{ID: "third", KnowledgeID: "doc-c", Score: 0.7},
},
},
}
plugin := &PluginFilterTopK{}
err := plugin.OnEvent(
context.Background(),
types.FILTER_TOP_K,
chatManage,
func() *PluginError { return nil },
)
require.Nil(t, err)
require.Len(t, chatManage.MergeResult, 2)
assert.Equal(t, types.RetrievalTruncation{
Stage: types.RetrievalStageRerank,
Candidates: 40,
}, *chatManage.Truncation)
}
// A prompt that saw every candidate must not carry the subset caveat, so the
// field stays nil when the result count is at or below topK.
func TestPluginFilterTopKLeavesTruncationNilWhenNothingDropped(t *testing.T) {
for _, topK := range []int{2, 5} {
t.Run(fmt.Sprintf("topK=%d", topK), func(t *testing.T) {
chatManage := &types.ChatManage{
PipelineRequest: types.PipelineRequest{RerankTopK: topK},
PipelineState: types.PipelineState{
MergeResult: []*types.SearchResult{
{ID: "first", KnowledgeID: "doc-a", Score: 0.9},
{ID: "second", KnowledgeID: "doc-b", Score: 0.8},
},
},
}
plugin := &PluginFilterTopK{}
err := plugin.OnEvent(
context.Background(),
types.FILTER_TOP_K,
chatManage,
func() *PluginError { return nil },
)
require.Nil(t, err)
require.Len(t, chatManage.MergeResult, 2)
assert.Nil(t, chatManage.Truncation)
})
}
}
func searchResultIDs(results []*types.SearchResult) []string {
ids := make([]string, 0, len(results))
for _, result := range results {

View File

@ -120,6 +120,13 @@ func (p *PluginMerge) selectInputResults(ctx context.Context, chatManage *types.
kept = append(kept, r)
}
}
// This cut already hides candidates from the prompt, so record it
// here: with rerank skipped there is nothing left for FILTER_TOP_K to
// cut, and without the record the model would read a subset as the
// complete result set.
if len(kept) < len(result) {
chatManage.RecordRetrievalCut(types.RetrievalStageMerge, len(result))
}
result = kept
}
pipelineWarn(ctx, "Merge", "fallback", map[string]interface{}{

View File

@ -61,12 +61,22 @@ func prepareMessagesWithModelContext(
registry.RegisterSearchResults(knowledgeResults)
var contextParts []string
if len(knowledgeRows) > 0 {
data := map[string]interface{}{
"display_type": "search_results",
"results": knowledgeRows,
}
// The ranked filter chain (rerank → merge → FILTER_TOP_K) may have
// dropped candidates before this view was built. Carry the recorded
// pool and the number of knowledge passages actually rendered, so the
// model context can say the passages are a subset without restating
// the pool as a knowledge-base match count.
if truncation := chatManage.Truncation; truncation != nil {
data["retrieval_shown"] = len(knowledgeRows)
data["retrieval_candidates"] = truncation.Candidates
}
contextParts = append(contextParts, registry.ModelToolResult(&types.ToolResult{
Success: true,
Data: map[string]interface{}{
"display_type": "search_results",
"results": knowledgeRows,
},
Data: data,
}))
}
if len(webRows) > 0 {

View File

@ -137,3 +137,160 @@ func TestPrepareMessagesWithModelContextSuppressesCitationsWhenDisabled(t *testi
require.NotContains(t, messages[1].Content, "chunk-1")
require.Equal(t, "answer ", refs.DecodeOutputText(`answer <ref id="c1"/>`))
}
// A truncated ranked list must reach the model context view: this is the text
// the model actually reads, and RenderedContexts is replaced wholesale. N is
// the number of passages this view renders, M the candidate pool the first
// cutting stage received.
func TestPrepareMessagesWithModelContextReportsTruncatedRetrieval(t *testing.T) {
manage := &types.ChatManage{
PipelineRequest: types.PipelineRequest{
Query: "who holds the CCSK certificate",
SummaryConfig: types.SummaryConfig{Prompt: "system"},
},
PipelineState: types.PipelineState{
UserContent: "question",
MergeResult: []*types.SearchResult{{
ID: "chunk-1", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1",
KnowledgeTitle: "Doc", Content: "evidence",
}},
Truncation: &types.RetrievalTruncation{
Stage: types.RetrievalStageRerank,
Candidates: 146,
},
},
}
messages, _ := prepareMessagesWithModelContext(context.Background(), manage)
require.Contains(t, messages[1].Content, `<subset shown="1" candidates="146">`)
require.Contains(t, messages[1].Content, "state that the provided context contains only the top 1 of 146")
require.Contains(t, messages[1].Content, "otherwise answer normally without adding a disclaimer")
manage.Truncation = nil
messages, _ = prepareMessagesWithModelContext(context.Background(), manage)
require.NotContains(t, messages[1].Content, "<subset")
}
// End to end: FILTER_TOP_K drops candidates, INTO_CHAT_MESSAGE renders the
// passages, and the caveat must land in the messages the model reads — not in
// RenderedContexts, which this stage replaces wholesale.
func TestTruncatedRetrievalReachesModelMessages(t *testing.T) {
manage := &types.ChatManage{
PipelineRequest: types.PipelineRequest{
Query: "who holds the CCSK certificate",
RerankTopK: 2,
SummaryConfig: types.SummaryConfig{Prompt: "system", ContextTemplate: "{{query}}\n{{contexts}}"},
},
PipelineState: types.PipelineState{
MergeResult: []*types.SearchResult{
{ID: "c1", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1", Content: "first", Score: 0.9},
{ID: "c2", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1", Content: "second", Score: 0.8},
{ID: "c3", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1", Content: "third", Score: 0.7},
},
},
}
next := func() *PluginError { return nil }
filter := &PluginFilterTopK{}
if err := filter.OnEvent(context.Background(), types.FILTER_TOP_K, manage, next); err != nil {
t.Fatalf("filter_top_k: %v", err)
}
intoChat := &PluginIntoChatMessage{}
if err := intoChat.OnEvent(context.Background(), types.INTO_CHAT_MESSAGE, manage, next); err != nil {
t.Fatalf("into_chat_message: %v", err)
}
require.Len(t, manage.MergeResult, 2)
require.NotContains(t, manage.RenderedContexts, "<subset")
messages, _ := prepareMessagesWithModelContext(context.Background(), manage)
require.Contains(t, messages[1].Content, `<subset shown="2" candidates="3">`)
require.Contains(t, messages[1].Content, "second")
require.NotContains(t, messages[1].Content, "third")
}
// End to end on the rerank path: with a rerank model the stage that cuts is
// rerank, not FILTER_TOP_K, which finds the already cut list and does nothing.
// Without the rerank record this turn reaches the prompt with no caveat at
// all, and a count question answers from the two shown passages as if they
// were every match.
func TestRerankTruncationReachesModelMessages(t *testing.T) {
manage := &types.ChatManage{
PipelineRequest: types.PipelineRequest{
Query: "how many certificates do we hold",
RerankModelID: "rr-1",
RerankTopK: 2,
RerankThreshold: 0.3,
SummaryConfig: types.SummaryConfig{Prompt: "system", ContextTemplate: "{{query}}\n{{contexts}}"},
},
PipelineState: types.PipelineState{
RewriteQuery: "how many certificates do we hold",
SearchResult: []*types.SearchResult{
{ID: "c1", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1", Content: "first", Score: 0.5},
{ID: "c2", KnowledgeID: "doc-2", KnowledgeBaseID: "kb-1", Content: "second", Score: 0.4},
{ID: "c3", KnowledgeID: "doc-3", KnowledgeBaseID: "kb-1", Content: "third", Score: 0.3},
{ID: "c4", KnowledgeID: "doc-4", KnowledgeBaseID: "kb-1", Content: "fourth", Score: 0.2},
},
},
}
next := func() *PluginError { return nil }
rerank := &PluginRerank{modelService: &rerankOnlyModelService{
reranker: &fixedReranker{scores: []float64{0.9, 0.8, 0.7, 0.6}},
}}
if err := rerank.OnEvent(context.Background(), types.CHUNK_RERANK, manage, next); err != nil {
t.Fatalf("chunk_rerank: %v", err)
}
merge := &PluginMerge{}
if err := merge.OnEvent(context.Background(), types.CHUNK_MERGE, manage, next); err != nil {
t.Fatalf("chunk_merge: %v", err)
}
filter := &PluginFilterTopK{}
if err := filter.OnEvent(context.Background(), types.FILTER_TOP_K, manage, next); err != nil {
t.Fatalf("filter_top_k: %v", err)
}
require.Len(t, manage.MergeResult, 2)
require.NotNil(t, manage.Truncation, "rerank dropped 2 of 4 candidates and must record it")
require.Equal(t, types.RetrievalTruncation{
Stage: types.RetrievalStageRerank,
Candidates: 4,
}, *manage.Truncation)
messages, _ := prepareMessagesWithModelContext(context.Background(), manage)
require.Contains(t, messages[1].Content, `<subset shown="2" candidates="4">`)
}
// End to end on the merge fallback path: rerank produced nothing, merge cut
// the retrieval list to RerankTopK, and FILTER_TOP_K has nothing left to cut.
// The prompt must still carry the record of the earlier cut.
func TestMergeFallbackTruncationReachesModelMessages(t *testing.T) {
manage := &types.ChatManage{
PipelineRequest: types.PipelineRequest{
Query: "how many certificates do we hold",
RerankTopK: 2,
SummaryConfig: types.SummaryConfig{Prompt: "system", ContextTemplate: "{{query}}\n{{contexts}}"},
},
PipelineState: types.PipelineState{
SearchResult: []*types.SearchResult{
{ID: "c1", KnowledgeID: "doc-1", KnowledgeBaseID: "kb-1", Content: "first", Score: 0.9},
{ID: "c2", KnowledgeID: "doc-2", KnowledgeBaseID: "kb-1", Content: "second", Score: 0.8},
{ID: "c3", KnowledgeID: "doc-3", KnowledgeBaseID: "kb-1", Content: "third", Score: 0.7},
},
},
}
next := func() *PluginError { return nil }
merge := &PluginMerge{}
if err := merge.OnEvent(context.Background(), types.CHUNK_MERGE, manage, next); err != nil {
t.Fatalf("chunk_merge: %v", err)
}
filter := &PluginFilterTopK{}
if err := filter.OnEvent(context.Background(), types.FILTER_TOP_K, manage, next); err != nil {
t.Fatalf("filter_top_k: %v", err)
}
require.Len(t, manage.MergeResult, 2)
require.NotNil(t, manage.Truncation, "merge dropped 1 of 3 candidates and must record it")
require.Equal(t, types.RetrievalTruncation{
Stage: types.RetrievalStageMerge,
Candidates: 3,
}, *manage.Truncation)
messages, _ := prepareMessagesWithModelContext(context.Background(), manage)
require.Contains(t, messages[1].Content, `<subset shown="2" candidates="3">`)
}

View File

@ -148,6 +148,20 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
})
return ErrSearchNothing
}
// Rerank drops candidates before FILTER_TOP_K ever runs: threshold
// rejection, the MaxCandidates cap and the MMR top-k all hide passage
// candidates that retrieval produced. Record the cut from the pool the
// stage received so the prompt can say its passages are a subset, whether
// or not FILTER_TOP_K finds anything left to cut.
if dropped := len(chatManage.SearchResult) - len(chatManage.RerankResult); dropped > 0 {
chatManage.RecordRetrievalCut(types.RetrievalStageRerank, len(chatManage.SearchResult))
pipelineInfo(ctx, "Rerank", "truncation", map[string]interface{}{
"stage": types.RetrievalStageRerank,
"candidates": len(chatManage.SearchResult),
"kept": len(chatManage.RerankResult),
"dropped": dropped,
})
}
pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{
"filtered_cnt": len(chatManage.RerankResult),
"outcome": diag.Outcome,

View File

@ -224,3 +224,88 @@ func TestMergeFallbackKeepsRerankTopK(t *testing.T) {
t.Fatalf("fallback input = %+v", got)
}
}
// Rerank drops candidates before FILTER_TOP_K ever runs — threshold
// rejection, the candidate cap and the MMR top-k all hide passages. The cut
// has to be recorded here, or a turn with a rerank model reaches the prompt
// with no record that its context is a subset, and a count question reads the
// shown passages as the whole result set.
func TestPluginRerankRecordsEarlyTruncation(t *testing.T) {
cm := &types.ChatManage{
PipelineRequest: types.PipelineRequest{RerankModelID: "rr-1", RerankTopK: 2, RerankThreshold: 0.3},
PipelineState: types.PipelineState{
RewriteQuery: "q",
SearchResult: []*types.SearchResult{
{ID: "c1", Content: "first", Score: 0.5},
{ID: "c2", Content: "second", Score: 0.4},
{ID: "c3", Content: "third", Score: 0.3},
{ID: "c4", Content: "fourth", Score: 0.2},
},
},
}
plugin := &PluginRerank{modelService: &rerankOnlyModelService{
reranker: &fixedReranker{scores: []float64{0.9, 0.8, 0.7, 0.6}},
}}
err := plugin.OnEvent(context.Background(), types.CHUNK_RERANK, cm, func() *PluginError { return nil })
if err != nil {
t.Fatalf("OnEvent: %v", err)
}
if len(cm.RerankResult) != 2 {
t.Fatalf("rerank result = %d, want 2", len(cm.RerankResult))
}
want := types.RetrievalTruncation{Stage: types.RetrievalStageRerank, Candidates: 4}
if cm.Truncation == nil || *cm.Truncation != want {
t.Fatalf("truncation = %+v, want %+v", cm.Truncation, want)
}
}
// A rerank that returned every candidate it received did not hide anything,
// so the prompt must stay free of the subset caveat.
func TestPluginRerankKeepsAllCandidatesWithoutTruncation(t *testing.T) {
cm := rerankChatManage(0.3)
plugin := &PluginRerank{modelService: &rerankOnlyModelService{
reranker: &fixedReranker{scores: []float64{0.9, 0.8}},
}}
err := plugin.OnEvent(context.Background(), types.CHUNK_RERANK, cm, func() *PluginError { return nil })
if err != nil {
t.Fatalf("OnEvent: %v", err)
}
if len(cm.RerankResult) != 2 || cm.Truncation != nil {
t.Fatalf("rerank result = %d, truncation = %+v", len(cm.RerankResult), cm.Truncation)
}
}
// The merge fallback hides candidates too, and with rerank skipped it is the
// last stage that can say so: FILTER_TOP_K sees the already cut list and finds
// nothing left to truncate.
func TestMergeFallbackRecordsTruncation(t *testing.T) {
cm := &types.ChatManage{
PipelineRequest: types.PipelineRequest{RerankTopK: 2},
PipelineState: types.PipelineState{SearchResult: []*types.SearchResult{
{ID: "low", Score: 0.1}, {ID: "high", Score: 0.9}, {ID: "mid", Score: 0.5},
}},
}
(&PluginMerge{}).selectInputResults(context.Background(), cm)
want := types.RetrievalTruncation{Stage: types.RetrievalStageMerge, Candidates: 3}
if cm.Truncation == nil || *cm.Truncation != want {
t.Fatalf("truncation = %+v, want %+v", cm.Truncation, want)
}
}
// Graph hits survive the fallback cut, so a cut that only skipped graph hits
// dropped nothing the prompt would miss and must not raise the subset caveat.
func TestMergeFallbackWithoutLossRecordsNothing(t *testing.T) {
cm := &types.ChatManage{
PipelineRequest: types.PipelineRequest{RerankTopK: 1},
PipelineState: types.PipelineState{SearchResult: []*types.SearchResult{
{ID: "high", Score: 0.9},
{ID: "graph", Score: 0, MatchType: types.MatchTypeGraph},
}},
}
got := (&PluginMerge{}).selectInputResults(context.Background(), cm)
if len(got) != 2 || cm.Truncation != nil {
t.Fatalf("fallback input = %d, truncation = %+v", len(got), cm.Truncation)
}
}

View File

@ -377,12 +377,21 @@ func graphTruncationNote(data map[string]interface{}, shownRelations int) string
// annotateSearchNotes tells the model what a search result does not show:
// how many lower-ranked results were left out to fit the tool output budget
// (so it narrows the query or lowers the limit instead of concluding nothing
// else matched), and which knowledge bases could not be searched (so a
// failure is not read as an absence of evidence).
// else matched), how much of the ranked candidate pool the rendered passages
// stand for, and which knowledge bases could not be searched (so a failure is
// not read as an absence of evidence).
func annotateSearchNotes(output string, data map[string]interface{}) string {
omitted := intValue(data, "omitted_for_budget")
failures := stringSliceValue(data["partial_failures"])
if (omitted <= 0 && len(failures) == 0) || !strings.HasSuffix(output, "</retrieval>") {
// retrieval_candidates is the candidate pool of the first ranked-filter
// stage that dropped anything (rerank, merge or FILTER_TOP_K), not the
// number of matches in the knowledge base, which retrieval never computes
// at that depth; retrieval_shown is how many knowledge passages this view
// renders out of that pool. Both count passages.
shown := intValue(data, "retrieval_shown")
candidates := intValue(data, "retrieval_candidates")
truncated := shown > 0 && candidates > shown
if (omitted <= 0 && len(failures) == 0 && !truncated) || !strings.HasSuffix(output, "</retrieval>") {
return output
}
var b strings.Builder
@ -390,6 +399,18 @@ func annotateSearchNotes(output string, data map[string]interface{}) string {
fmt.Fprintf(&b, " <omitted count=\"%d\" reason=\"output_budget\">Lower-ranked results were left out to "+
"fit the output size. Narrow the query or lower limit to see them.</omitted>\n", omitted)
}
if truncated {
// The caveat is conditional on purpose: a truncated view is the norm
// for a ranked retrieval, so an unconditional "the answer may be
// incomplete" would add a disclaimer to ordinary factual answers that
// the subset does not affect. Only counting and exhaustive-list
// questions have to report the scope of what was read.
fmt.Fprintf(&b, " <subset shown=\"%d\" candidates=\"%d\">If the question asks for a count or an "+
"exhaustive list, state that the provided context contains only the top %d of %d candidate "+
"passages at this filtering stage, and do not treat these passage counts as the requested "+
"total; otherwise answer normally without adding a disclaimer solely because of this "+
"truncation.</subset>\n", shown, candidates, shown, candidates)
}
for _, failure := range failures {
fmt.Fprintf(&b, " <partial_failure>%s — these knowledge bases were not searched.</partial_failure>\n",
escapeText(failure))

View File

@ -22,6 +22,46 @@ func TestAnnotateSearchNotes(t *testing.T) {
}
}
// The subset note is conditional: only a counting or exhaustive-list question
// has to report the scope of the view. It must also carry the counts it was
// given, so the model reads the real shown / candidate numbers rather than a
// placeholder or a knowledge-base total.
func TestAnnotateSearchNotesReportsTruncatedRetrieval(t *testing.T) {
t.Parallel()
base := "<retrieval mode=\"hybrid\">\n</retrieval>"
got := annotateSearchNotes(base, map[string]interface{}{
"retrieval_shown": 5,
"retrieval_candidates": 146,
})
want := ` <subset shown="5" candidates="146">If the question asks for a count or an exhaustive list, ` +
`state that the provided context contains only the top 5 of 146 candidate passages at this ` +
`filtering stage, and do not treat these passage counts as the requested total; otherwise ` +
`answer normally without adding a disclaimer solely because of this truncation.</subset>`
if !strings.Contains(got, want) {
t.Fatalf("annotated = %q\nwant subset %q", got, want)
}
if strings.Contains(got, "say the answer may be incomplete") {
t.Fatalf("the unconditional disclaimer must be gone: %q", got)
}
if !strings.HasSuffix(got, "</retrieval>") {
t.Fatalf("subset note must stay inside the retrieval block: %q", got)
}
// A prompt that saw every candidate, or was given no usable counts,
// carries no caveat at all.
for name, data := range map[string]map[string]interface{}{
"complete": {"retrieval_shown": 5, "retrieval_candidates": 5},
"no counts": {},
"shown only": {"retrieval_shown": 5},
"candidates only": {"retrieval_candidates": 146},
"more than pool": {"retrieval_shown": 7, "retrieval_candidates": 5},
"no shown passage": {"retrieval_shown": 0, "retrieval_candidates": 146},
} {
if annotateSearchNotes(base, data) != base {
t.Fatalf("%s: a non-truncated retrieval must leave the output unchanged", name)
}
}
}
func TestAnnotateGraphResult(t *testing.T) {
t.Parallel()
base := "<retrieval mode=\"graph\">\n</retrieval>"

View File

@ -115,6 +115,52 @@ func (i QueryIntent) NeedsKBRetrieval() bool {
}
}
// Retrieval stage names recorded on RetrievalTruncation.Stage, in pipeline
// order.
const (
// RetrievalStageRerank is the CHUNK_RERANK stage, which drops candidates
// through threshold rejection, its candidate cap or its MMR top-k.
RetrievalStageRerank = "rerank"
// RetrievalStageMerge is the CHUNK_MERGE fallback cut to RerankTopK, taken
// when rerank produced nothing to merge.
RetrievalStageMerge = "merge"
// RetrievalStageFilterTopK is the FILTER_TOP_K stage.
RetrievalStageFilterTopK = "filter_top_k"
)
// RetrievalTruncation records that the passages in the model context are a
// subset of this turn's ranked candidates.
//
// Stage is the first pipeline stage that dropped candidates and Candidates is
// the number of candidate passages that stage received, so the pair always
// describes one population: the retrieval set that entered the ranked filter
// chain (rerank → merge → FILTER_TOP_K). Candidates is a pipeline-local count
// of retrieved passages — never the number of matches in the knowledge base,
// which retrieval never computes — and a later, narrower cut never replaces
// it. How many of those candidates the prompt actually shows is counted where
// the prompt is built, so the caveat is only emitted when the context really
// is a subset.
type RetrievalTruncation struct {
Stage string `json:"stage"`
Candidates int `json:"candidates"`
}
// RecordRetrievalCut notes that a ranked-list stage dropped candidate passages
// before the model context was built. candidates is the population that stage
// received, read before the cut; it must never be a knowledge-base match count,
// which retrieval does not compute at this depth.
//
// The first cut wins. Later stages only see an equal or narrower population
// (FILTER_TOP_K cuts merge output, merge's fallback cuts the search output), so
// letting them overwrite the record would restate the candidate pool as
// something smaller than what retrieval produced and hide how much was dropped.
func (c *ChatManage) RecordRetrievalCut(stage string, candidates int) {
if c == nil || c.Truncation != nil || candidates <= 0 {
return
}
c.Truncation = &RetrievalTruncation{Stage: stage, Candidates: candidates}
}
// PipelineState holds mutable intermediate data that plugins read and write
// as the pipeline progresses.
type PipelineState struct {
@ -138,6 +184,11 @@ type PipelineState struct {
ImageDescription string `json:"-"`
QuotedContext string `json:"-"` // Quoted message text, injected at LLM prompt stage
SystemPromptOverride string `json:"-"`
// Truncation records the first ranked-list cut this turn: the stage that
// dropped candidates and the size of the candidate pool it cut. Nil when
// no stage dropped anything, so a prompt only carries the caveat when its
// context really is a subset.
Truncation *RetrievalTruncation `json:"-"`
// MemoryPrompt is the long-term memory envelope appended to the system
// prompt for this turn, empty when memory is off or nothing matched.
MemoryPrompt string `json:"-"`