From b60351f86128d22bfa998b13989530917d32ef2b Mon Sep 17 00:00:00 2001 From: lyingbug Date: Thu, 10 Sep 2026 00:31:43 +0800 Subject: [PATCH] fix(agent): prevent MCP answers from citing unrelated KB sources (#3144) * fix(agent): keep MCP citations separate from historical KB sources * style(modelcontext): wrap citation prompts and regression tests * test(service): align fallback citations with current evidence policy --- internal/agent/engine_test.go | 36 +++++ internal/agent/observe.go | 2 +- .../service/fallback_messages_test.go | 21 +-- internal/modelcontext/README.md | 18 +++ internal/modelcontext/citations.go | 33 +++-- internal/modelcontext/mcp_sources.go | 69 +++++++++ internal/modelcontext/mcp_sources_test.go | 135 ++++++++++++++++++ internal/modelcontext/model_output.go | 12 +- internal/modelcontext/registry.go | 12 ++ internal/modelcontext/sources.go | 40 ++++-- internal/modelcontext/sources_test.go | 6 + 11 files changed, 352 insertions(+), 32 deletions(-) create mode 100644 internal/modelcontext/mcp_sources.go create mode 100644 internal/modelcontext/mcp_sources_test.go diff --git a/internal/agent/engine_test.go b/internal/agent/engine_test.go index 41789a57..229f0b00 100644 --- a/internal/agent/engine_test.go +++ b/internal/agent/engine_test.go @@ -142,6 +142,42 @@ func TestStreamLLMSummarySlugSurvivesDocumentCompaction(t *testing.T) { require.NotContains(t, result.ToolCalls[0].Function.Arguments, "res://") } +// Reproduce an MCP-only turn following a property-management answer, with +// unrelated FAQ entries injected by the bound-KB directory. +func TestStreamMCPAnswerRejectsUnretrievedKnowledgeCitations(t *testing.T) { + model := &mockChat{responses: []mockResponse{{chunks: []types.StreamResponse{ + {ResponseType: types.ResponseTypeAnswer, Content: `KM文章 正确来源`, Done: true, FinishReason: "stop"}, + }}}} + engine := newTestEngine(t, model) + engine.knowledgeBasesInfo = []*KnowledgeBaseInfo{{ + ID: "faq-kb", Name: "FAQ TEST", Type: "faq", RecentDocs: []RecentDocInfo{ + {ChunkID: "faq-1", Title: "什么是 WeKnora?", FAQStandardQuestion: "什么是 WeKnora?"}, + {ChunkID: "faq-2", Title: "如何创建知识库?", FAQStandardQuestion: "如何创建知识库?"}, + }, + }} + userTurn := engine.RenderUserTurnContent("session", "KM上有趣的事情") + const article = "https://km.woa.com/articles/show/669504?jumpfrom=kmmcp" + toolResult := engine.modelContext.ModelToolResultForTool("call_mcp_tool", &types.ToolResult{ + Success: true, Output: "标题: AI玩法\n摘要: Computer Use 案例\n链接: " + article, + }) + var emitted strings.Builder + result, err := engine.streamLLMToEventBus(context.Background(), []chat.Message{ + {Role: "assistant", Content: `物业工作`}, + {Role: "user", Content: userTurn}, + {Role: "tool", Name: "call_mcp_tool", Content: toolResult}, + }, nil, func(chunk *types.StreamResponse, _ string) { + emitted.WriteString(chunk.Content) + }) + require.NoError(t, err) + want := `KM文章 正确来源` + require.Equal(t, want, result.Content) + require.Equal(t, want, emitted.String(), "invalid references must not reach SSE even transiently") + require.Contains(t, model.calls[0][2].Content, ``, + Query: "previous", + Answer: `Previous ` + + ``, }} messages, refs := prepareFallbackMessages(cm, "legacy fallback prompt") require.Contains(t, messages[0].Content, "Source handling protocol") - require.Equal(t, `Previous `, messages[2].Content) - require.Equal(t, - ``, - refs.DecodeOutputText(``), - ) + require.Equal(t, `Previous `, messages[2].Content) + // Fallback has no current retrieval evidence. Historical handles are kept + // for navigation but must not authorize citations in the new answer. + raw := `Answer ` + require.Equal(t, "Answer ", refs.DecodeOutputText(raw)) + decoder := refs.StreamDecoder() + streamed := decoder.Feed(`Answer `) + decoder.Flush() + require.Equal(t, "Answer ", streamed) } func TestPrepareFallbackMessagesSuppressesCitationsWhenDisabled(t *testing.T) { diff --git a/internal/modelcontext/README.md b/internal/modelcontext/README.md index b8893b1d..ad466885 100644 --- a/internal/modelcontext/README.md +++ b/internal/modelcontext/README.md @@ -15,6 +15,7 @@ codec lives in this one package: | `stream.go` | `streamHold` suffix-hold primitive and every streaming decoder | | `tool_policy.go` | the single policy layer: per-tool key contracts + `sourceKeySpaces` dispatch | | `mcp.go` | MCP bridge routing handles (`msN`/`mtN`), definition projection and envelope codec | +| `mcp_sources.go` | bounded citation sidecar for HTTP(S) links in successful MCP results | | `handles.go` | exported `HandleTable` for invocation-local spaces (`iN`, `ref-N`, `c000`) | ## Identity rules @@ -46,6 +47,14 @@ The registry owns codec ordering. Resource references are encoded before source IDs so a Wiki slug such as `summary/` cannot be corrupted into `summary/d1`. +Source addressability is separate from citation eligibility. Use +`RegisterContextChunk` for bound-KB directory entries. Replayed citations, +legacy tool history, and tool arguments also register navigation handles only. +`ModelToolResultForTool` and explicitly supplied retrieval results authorize +citations for the current execution; rereading a historical source promotes the +same handle without renumbering it. Both complete and streaming decoders drop +references that have not been backed by current evidence. + ## Observability Langfuse generation observations contain the exact encoded payload sent to and @@ -77,6 +86,15 @@ still use the normal `res://NNNN` codec. A future MCP ID mapping must be an explicit server/tool annotation and should plug into this registry rather than create a parallel mapper. +Successful MCP execution results additionally receive an +`external_source_candidates` sidecar with up to 50 distinct observed HTTP(S) +links mapped to `wN`. The external payload is not rewritten, discovery schemas +are not scanned, and article IDs are never used to guess URLs. A candidate link +does not establish that its page was read: the model must select the link whose +associated result supports the claim. When no handle is supplied, the citation +protocol permits an exact supplied source URL as a Markdown link, never a +borrowed knowledge-base citation. + The application-owned MCP bridge is an explicit exception for routing fields: `discover_mcp_tools.server_id` uses `msN`, and `call_mcp_tool.tool_ref` uses `mtN`. Tool definition enums, service summaries, discovery results and replayed diff --git a/internal/modelcontext/citations.go b/internal/modelcontext/citations.go index 34b73f06..27f0808a 100644 --- a/internal/modelcontext/citations.go +++ b/internal/modelcontext/citations.go @@ -20,7 +20,15 @@ Retrieved content uses request-local source handles: cN identifies a knowledge c const citationEnabledProtocolPrompt = ` - Source citations are enabled for this answer. Cite a knowledge chunk with exactly and a web page with exactly . -- Copy only cN/wN handles that appeared in supplied context or tool results. Never cite dN/bN. +- Cite only cN/wN handles backed by tool results for the current task, and only when that source supports the adjacent + claim. Never cite dN/bN. +- Handles in historical answers, tool arguments, or the bound knowledge-base directory are for navigation, not current + evidence. Retrieve the relevant source before citing it. +- MCP results are external sources. Use the wN handle for the matching URL in the system-provided + external_source_candidates list; never substitute a knowledge-base cN handle for MCP content. Candidate URLs are links + observed in the result, not proof that every linked page was read. +- If a source has no citation handle, use its exact supplied HTTP(S) URL as a Markdown link when available. + If neither is available, omit the citation; never invent or borrow a source. - Never output or tags yourself; the system expands valid tags after generation. - Keep each inline on the same line as the claim it supports. Do not group citations at the end. - These rules supersede earlier, saved, or custom prompt instructions about citation syntax.` @@ -60,7 +68,7 @@ var ( knowledgeTitleAttrRE = regexp.MustCompile(`(?i)\bknowledge_title\s*=\s*"([^"]*)"`) ) -func (r *sourceRegistry) registerLegacyToolReferences(text string) { +func (r *sourceRegistry) registerLegacyToolReferences(text string, evidence bool) { if r == nil || text == "" { return } @@ -70,19 +78,19 @@ func (r *sourceRegistry) registerLegacyToolReferences(text string) { if chunkID == "" { continue } - r.RegisterChunk(ChunkReference{ + r.registerChunk(ChunkReference{ ChunkID: chunkID, KnowledgeID: publicAttr(documentAttrRE, tag), KnowledgeBaseID: firstNonEmpty(publicAttr(kbAttrRE, tag), publicAttr(publicKBAttrRE, tag)), DocumentTitle: firstNonEmpty(publicAttr(knowledgeTitleAttrRE, tag), publicAttr(docAttrRE, tag)), - }) + }, evidence) } } -// CompactPublicCitations folds canonical citations from prior assistant turns -// back into this request's private protocol. This prevents durable chunk IDs -// and web URLs in conversation history from becoming model-visible again. -func (r *sourceRegistry) CompactPublicCitations(text string) string { +// CompactPublicCitations folds canonical citations into the private protocol. +// Historical citations register navigation handles only; citations returned by +// a successful current source tool can also authorize evidence. +func (r *sourceRegistry) CompactPublicCitations(text string, evidence bool) string { if r == nil || text == "" { return text } @@ -91,11 +99,11 @@ func (r *sourceRegistry) CompactPublicCitations(text string) string { if chunkID == "" { return tag } - handle := r.RegisterChunk(ChunkReference{ + handle := r.registerChunk(ChunkReference{ ChunkID: chunkID, KnowledgeBaseID: publicAttr(publicKBAttrRE, tag), DocumentTitle: publicAttr(docAttrRE, tag), - }) + }, evidence) return `` }) return publicWebTagRE.ReplaceAllStringFunc(text, func(tag string) string { @@ -103,7 +111,7 @@ func (r *sourceRegistry) CompactPublicCitations(text string) string { if rawURL == "" { return tag } - handle := r.RegisterWeb(rawURL, publicAttr(titleAttrRE, tag)) + handle := r.registerWeb(rawURL, publicAttr(titleAttrRE, tag), evidence) return `` }) } @@ -172,6 +180,9 @@ func (r *sourceRegistry) ExpandText(text string) string { return "" } handle := strings.ToLower(match[1]) + if _, citable := r.citable.Load(handle); !citable { + return "" + } if chunkID, chunkRef, ok := r.chunks.resolve(handle); ok { attrs := fmt.Sprintf(`doc="%s" chunk_id="%s"`, escapeAttr(chunkRef.DocumentTitle), escapeAttr(chunkID)) if chunkRef.KnowledgeBaseID != "" { diff --git a/internal/modelcontext/mcp_sources.go b/internal/modelcontext/mcp_sources.go new file mode 100644 index 00000000..898254c1 --- /dev/null +++ b/internal/modelcontext/mcp_sources.go @@ -0,0 +1,69 @@ +package modelcontext + +import ( + "encoding/json" + "fmt" + "html" + "net/url" + "regexp" + "strings" +) + +// MCP schemas and payloads remain opaque. This sidecar only indexes literal +// HTTP(S) links observed in a successful result; it never interprets external +// IDs as knowledge chunks or synthesizes URLs from article IDs. +const maxMCPSourceCandidates = 50 + +var ( + mcpJSONStringRE = regexp.MustCompile(`"(?:[^"\\]|\\.)*"`) + mcpURLRE = regexp.MustCompile("https?://[^\\s<>\"'`\\\\,。;!?、()【】]+") +) + +func (r *Registry) mcpSourceCandidates(output string) string { + if !r.sources.citationsEnabled { + return "" + } + // Decode JSON string escaping only in a scratch copy used to find links. + // Both the external body and the durable ToolResult stay unchanged. + scan := mcpJSONStringRE.ReplaceAllStringFunc(output, func(token string) string { + var value string + if json.Unmarshal([]byte(token), &value) == nil { + return `"` + value + `"` + } + return token + }) + seen := make(map[string]bool) + var rows strings.Builder + for _, match := range mcpURLRE.FindAllString(scan, -1) { + rawURL := strings.TrimRight(html.UnescapeString(match), ".,;!?") + // Strip Markdown/prose closing delimiters, retaining balanced URL + // parentheses such as /wiki/Function_(mathematics). + for _, pair := range [][2]string{{"(", ")"}, {"[", "]"}, {"{", "}"}} { + for strings.HasSuffix(rawURL, pair[1]) && strings.Count(rawURL, pair[1]) > strings.Count(rawURL, pair[0]) { + rawURL = strings.TrimSuffix(rawURL, pair[1]) + } + } + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Hostname() == "" || parsed.User != nil || + (parsed.Scheme != "https" && parsed.Scheme != "http") { + continue + } + key := canonicalWebURL(rawURL) + if seen[key] { + continue + } + seen[key] = true + handle := r.sources.RegisterWeb(rawURL, "") + fmt.Fprintf(&rows, "\n", handle, escapeAttr(rawURL)) + if len(seen) >= maxMCPSourceCandidates { + break + } + } + if rows.Len() == 0 { + return "" + } + return "\n\n\n" + + "System-indexed links from this MCP result. Cite the matching wN only when the result supports the claim; " + + "a link alone does not mean the linked page was read. Do not use KB cN handles for this external content.\n" + + rows.String() + "" +} diff --git a/internal/modelcontext/mcp_sources_test.go b/internal/modelcontext/mcp_sources_test.go new file mode 100644 index 00000000..a10f733b --- /dev/null +++ b/internal/modelcontext/mcp_sources_test.go @@ -0,0 +1,135 @@ +package modelcontext + +import ( + "fmt" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestMCPAnswerCannotBorrowDirectoryOrHistoricalKnowledgeCitations(t *testing.T) { + r := NewRegistry(true) + r.RegisterContextChunk(ChunkReference{ChunkID: "faq-1", DocumentTitle: "什么是 WeKnora?"}) + r.RegisterContextChunk(ChunkReference{ChunkID: "faq-2", DocumentTitle: "如何创建知识库?"}) + history := []chat.Message{{Role: "assistant", Content: `物业工作 `}} + r.EncodeMessages(history) + const article = "https://km.woa.com/articles/show/669504?jumpfrom=kmmcp" + result := &types.ToolResult{Success: true, Output: "标题: AI 玩法\nAI摘要: Computer Use 案例\n链接: " + article} + modelOutput := r.ModelToolResultForTool("call_mcp_tool", result) + require.True(t, strings.HasPrefix(modelOutput, result.Output), "external payload is preserved") + require.Contains(t, modelOutput, ``) + require.Equal(t, modelOutput, r.ModelToolResultForTool("call_mcp_tool", result), + "rebuilding context keeps stable handles") + r.EncodeMessages(history) // Replaying history must not promote its citations. + raw := `玩法 动画 对话 来源` + want := `玩法 动画 对话 来源` + require.Equal(t, want, r.DecodeOutputText(raw)) + decoder := r.StreamDecoder() + var streamed strings.Builder + for _, char := range raw { + streamed.WriteString(decoder.Feed(string(char))) + } + streamed.WriteString(decoder.Flush()) + require.Equal(t, want, streamed.String(), "SSE and final text apply identical evidence checks") + + // Directory handles remain usable to retrieve the actual FAQ. + calls := []types.LLMToolCall{{Function: types.FunctionCall{ + Name: "list_knowledge_chunks", Arguments: `{"faq_id":"c1"}`, + }}} + r.DecodeToolCalls(calls) + require.JSONEq(t, `{"faq_id":"faq-1"}`, calls[0].Function.Arguments) + r.ModelToolResultForTool("knowledge_search", &types.ToolResult{Success: true, Data: map[string]interface{}{ + "display_type": "search_results", + "results": []map[string]interface{}{{ + "chunk_id": "faq-1", "knowledge_title": "什么是 WeKnora?", "content": "知识库管理系统", + }}, + }}) + r.RegisterContextChunk(ChunkReference{ChunkID: "faq-1"}) + r.EncodeMessages(history) + require.Contains(t, r.DecodeOutputText(``), `chunk_id="faq-1"`, + "fresh KB evidence is still citable in a mixed-source answer") + require.Empty(t, r.DecodeOutputText(``)) +} + +func TestToolArgumentsAloneDoNotAuthorizeCitations(t *testing.T) { + r := NewRegistry(true) + r.EncodeMessages([]chat.Message{{Role: "assistant", ToolCalls: []chat.ToolCall{ + {Function: chat.FunctionCall{Name: "list_knowledge_chunks", Arguments: `{"faq_id":"faq-real"}`}}, + {Function: chat.FunctionCall{Name: "web_fetch", Arguments: `{"items":[{"url":"https://example.com/unread"}]}`}}, + }}}) + require.Empty(t, r.DecodeOutputText(``)) + r.ModelToolResultForTool("call_mcp_tool", &types.ToolResult{Success: false, Error: "https://example.com/error"}) + require.Empty(t, r.DecodeOutputText(``)) +} + +func TestCurrentLegacySourceResultsRemainCitableButHistoryDoesNot(t *testing.T) { + for _, output := range []string{ + `Evidence`, + `Evidence `, + } { + r := NewRegistry(true) + r.EncodeMessages([]chat.Message{{Role: "tool", Name: "wiki_read_page", Content: output}}) + require.Empty(t, r.DecodeOutputText(``)) + result := &types.ToolResult{Success: true, Output: output} + modelOutput := r.ModelToolResultForTool("wiki_read_page", result) + require.Contains(t, modelOutput, "c1") + require.Equal(t, output, result.Output) + require.Contains(t, r.DecodeOutputText(``), `chunk_id="chunk-real"`) + } +} + +func TestMCPSourceCandidatesPreserveExternalLinksAndPayload(t *testing.T) { + for _, tool := range []string{"call_mcp_tool", "mcp_external_search"} { + t.Run(tool, func(t *testing.T) { + r := NewRegistry(true) + body := "[MCP result]\n" + `{"url":"https:\/\/example.com\/article?a=1\u0026b=2","chunk_id":"foreign-id"}` + + "\n[Article](https://example.com/article?a=1&b=2)\n[Wiki](https://example.com/Function_(math))。\n" + + "[HTML](https://example.com/page?x=1&y=2)\n" + + "https://user:pass@example.com/private javascript:alert(1) https://" + result := &types.ToolResult{Success: true, Output: body} + got := r.ModelToolResultForTool(tool, result) + require.Equal(t, body, result.Output) + require.True(t, strings.HasPrefix(got, body)) + sidecar := strings.TrimPrefix(got, body) + require.Equal(t, 3, strings.Count(sidecar, "") + require.Empty(t, r.DecodeOutputText(``)) + } + r = NewRegistry(true) + require.Equal(t, "No links", r.ModelToolResultForTool("call_mcp_tool", &types.ToolResult{ + Success: true, Output: "No links", + })) +} diff --git a/internal/modelcontext/model_output.go b/internal/modelcontext/model_output.go index 615a096b..b0344395 100644 --- a/internal/modelcontext/model_output.go +++ b/internal/modelcontext/model_output.go @@ -23,6 +23,14 @@ func (r *sourceRegistry) ModelOutput(result *types.ToolResult) string { if result == nil { return "" } + // Legacy-format output from a current source tool is evidence too. Replay + // registration alone must never grant this eligibility. + if result.Success { + r.registerLegacyToolReferences(result.Output, true) + copyResult := *result + copyResult.Output = r.CompactPublicCitations(result.Output, true) + result = ©Result + } displayType := stringValue(result.Data, "display_type") if displayType == "web_fetch_results" { return r.modelWebFetchOutput(mapsValue(result.Data["results"]), result.Output) @@ -64,7 +72,7 @@ func (r *sourceRegistry) registerStructuredReferences(raw string) { walk = func(key string, value interface{}) { switch typed := value.(type) { case string: - r.registerSourceIDByKey(key, typed) + r.registerSourceIDByKey(key, typed, true) case []interface{}: for _, item := range typed { walk(key, item) @@ -82,7 +90,7 @@ func (r *sourceRegistry) modelDatabaseQueryOutput(rows []map[string]interface{}, for _, row := range rows { for key, raw := range row { if value, ok := raw.(string); ok { - r.registerSourceIDByKey(key, value) + r.registerSourceIDByKey(key, value, true) } } } diff --git a/internal/modelcontext/registry.go b/internal/modelcontext/registry.go index 744dd07e..f17e8310 100644 --- a/internal/modelcontext/registry.go +++ b/internal/modelcontext/registry.go @@ -208,6 +208,15 @@ func (r *Registry) RegisterChunk(ref ChunkReference) string { return r.sources.RegisterChunk(ref) } +// RegisterContextChunk makes a directory entry addressable by tools without +// allowing it to substantiate an answer before retrieval. +func (r *Registry) RegisterContextChunk(ref ChunkReference) string { + if r == nil || r.sources == nil { + return "" + } + return r.sources.registerChunk(ref, false) +} + func (r *Registry) RegisterDocument(id string) string { if r == nil || r.sources == nil { return "" @@ -292,6 +301,9 @@ func (r *Registry) ModelToolResultForTool(toolName string, result *types.ToolRes if sourceCompactionAllowed(toolName) { modelOutput = r.sources.CompactKnownText(modelOutput) } + if result.Success && (toolName == "call_mcp_tool" || strings.HasPrefix(toolName, "mcp_")) { + modelOutput += r.mcpSourceCandidates(result.Output) + } return r.resources.EncodeText(modelOutput) + outputFilesPrompt(result) } diff --git a/internal/modelcontext/sources.go b/internal/modelcontext/sources.go index 23953535..11a0d652 100644 --- a/internal/modelcontext/sources.go +++ b/internal/modelcontext/sources.go @@ -11,6 +11,7 @@ import ( "regexp" "sort" "strings" + "sync" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" @@ -34,6 +35,9 @@ type webMeta struct { // round). Handles are never persisted or accepted across requests. type sourceRegistry struct { citationsEnabled bool + // Addressable IDs from history, directory entries, and tool arguments are + // not evidence until a current tool result supplies the source. + citable sync.Map // handle -> true; registrations may run concurrently chunks *handleTable[ChunkReference] docs *handleTable[struct{}] @@ -74,6 +78,10 @@ func knownHandle[M any](table *handleTable[M], id string) string { } func (r *sourceRegistry) RegisterChunk(ref ChunkReference) string { + return r.registerChunk(ref, true) +} + +func (r *sourceRegistry) registerChunk(ref ChunkReference, evidence bool) string { if r == nil { return "" } @@ -84,7 +92,11 @@ func (r *sourceRegistry) RegisterChunk(ref ChunkReference) string { if shortSourceHandleRE.MatchString(ref.ChunkID) { return knownHandle(r.chunks, ref.ChunkID) } - return r.chunks.register(ref.ChunkID, ref.ChunkID, ref, mergeChunkReference) + handle := r.chunks.register(ref.ChunkID, ref.ChunkID, ref, mergeChunkReference) + if evidence { + r.citable.Store(handle, true) + } + return handle } func mergeChunkReference(dst *ChunkReference, src ChunkReference) { @@ -128,6 +140,10 @@ func (r *sourceRegistry) RegisterKnowledgeBase(id string) string { } func (r *sourceRegistry) RegisterWeb(rawURL, title string) string { + return r.registerWeb(rawURL, title, true) +} + +func (r *sourceRegistry) registerWeb(rawURL, title string, evidence bool) string { rawURL = strings.TrimSpace(rawURL) if r == nil || rawURL == "" { return "" @@ -137,11 +153,15 @@ func (r *sourceRegistry) RegisterWeb(rawURL, title string) string { } // Dedup on the canonical (fragment-stripped) URL while decoding back to // the raw URL the model was originally shown. - return r.webs.register(canonicalWebURL(rawURL), rawURL, webMeta{title: title}, func(dst *webMeta, src webMeta) { + handle := r.webs.register(canonicalWebURL(rawURL), rawURL, webMeta{title: title}, func(dst *webMeta, src webMeta) { if dst.title == "" && src.title != "" { dst.title = src.title } }) + if evidence { + r.citable.Store(handle, true) + } + return handle } func canonicalWebURL(raw string) string { @@ -277,14 +297,14 @@ func (r *sourceRegistry) EncodeMessagesWithPolicies( for i := range out { processToolResult := out[i].Role == "tool" && (resultPolicy == nil || resultPolicy(out[i].Name)) if out[i].Role == "assistant" || processToolResult { - out[i].Content = r.CompactPublicCitations(out[i].Content) - out[i].ReasoningContent = r.CompactPublicCitations(out[i].ReasoningContent) + out[i].Content = r.CompactPublicCitations(out[i].Content, false) + out[i].ReasoningContent = r.CompactPublicCitations(out[i].ReasoningContent, false) } if len(out[i].MultiContent) > 0 { out[i].MultiContent = append([]chat.MessageContentPart(nil), out[i].MultiContent...) for j := range out[i].MultiContent { if out[i].MultiContent[j].Type == "text" && (out[i].Role == "assistant" || processToolResult) { - out[i].MultiContent[j].Text = r.CompactPublicCitations(out[i].MultiContent[j].Text) + out[i].MultiContent[j].Text = r.CompactPublicCitations(out[i].MultiContent[j].Text, false) } } } @@ -301,7 +321,7 @@ func (r *sourceRegistry) EncodeMessagesWithPolicies( } for i := range out { if out[i].Role == "tool" && (resultPolicy == nil || resultPolicy(out[i].Name)) { - r.registerLegacyToolReferences(out[i].Content) + r.registerLegacyToolReferences(out[i].Content, false) out[i].Content = r.CompactKnownText(out[i].Content) } for j := range out[i].ToolCalls { @@ -426,7 +446,7 @@ func (r *sourceRegistry) registerToolArgumentValue(key string, value interface{} switch typed := value.(type) { case string: if allowed(strings.ToLower(key)) { - r.registerSourceIDByKey(key, typed) + r.registerSourceIDByKey(key, typed, false) } case []interface{}: for _, item := range typed { @@ -444,7 +464,7 @@ func (r *sourceRegistry) registerToolArgumentValue(key string, value interface{} // sourceKeySpaces — the same table that gates handle decode — so the recognized // key set (and the http/https guard for web references) cannot drift between // registration and decoding. -func (r *sourceRegistry) registerSourceIDByKey(key, value string) { +func (r *sourceRegistry) registerSourceIDByKey(key, value string, evidence bool) { value = strings.TrimSpace(value) if value == "" || shortSourceHandleRE.MatchString(value) { return @@ -455,7 +475,7 @@ func (r *sourceRegistry) registerSourceIDByKey(key, value string) { } switch space { case spaceChunk: - r.RegisterChunk(ChunkReference{ChunkID: value}) + r.registerChunk(ChunkReference{ChunkID: value}, evidence) case spaceDocument: r.RegisterDocument(value) case spaceDocumentRef: @@ -468,7 +488,7 @@ func (r *sourceRegistry) registerSourceIDByKey(key, value string) { // (res://, storage providers) must never enter the web handle space, // where CompactKnownText would rewrite them a second time. if parsed, err := url.Parse(value); err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") { - r.RegisterWeb(value, "") + r.registerWeb(value, "", evidence) } } } diff --git a/internal/modelcontext/sources_test.go b/internal/modelcontext/sources_test.go index 66e95289..ce1722f9 100644 --- a/internal/modelcontext/sources_test.go +++ b/internal/modelcontext/sources_test.go @@ -137,6 +137,10 @@ func TestEncodeMessagesCompactsCanonicalCitationsFromHistory(t *testing.T) { require.Equal(t, `Knowledge ; web `, encoded[0].Content) require.NotContains(t, encoded[0].Content, "chunk-real") require.NotContains(t, encoded[0].Content, "https://example.com") + require.Equal(t, " ", registry.ExpandText(` `), "history is not current evidence") + // A fresh retrieval promotes the same handles, retaining canonical metadata. + registry.RegisterChunk(ChunkReference{ChunkID: "chunk-real"}) + registry.RegisterWeb("https://example.com/a?x=1&y=2", "") require.Equal(t, ` `, registry.ExpandText(` `), @@ -174,6 +178,8 @@ func TestEncodeMessagesMigratesLegacyToolHistoryAtReadTime(t *testing.T) { require.Contains(t, encoded[1].Content, `knowledge_id="d1"`) require.Contains(t, encoded[1].Content, `knowledge_base_id="b1"`) require.Equal(t, `Legacy answer `, encoded[2].Content) + require.Empty(t, registry.ExpandText(``), "legacy tool history does not authorize new citations") + registry.RegisterChunk(ChunkReference{ChunkID: "chunk-real"}) require.Equal(t, ``, registry.ExpandText(``),