fix(agent): prevent MCP answers from citing unrelated KB sources (#3144)

* fix(agent): keep MCP citations separate from historical KB sources

* style(modelcontext): wrap citation prompts and regression tests

* test(service): align fallback citations with current evidence policy
main
lyingbug 2026-09-10 00:31:43 +08:00 committed by GitHub
parent 2d7df096ca
commit b60351f861
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 352 additions and 32 deletions

View File

@ -142,6 +142,42 @@ func TestStreamLLMSummarySlugSurvivesDocumentCompaction(t *testing.T) {
require.NotContains(t, result.ToolCalls[0].Function.Arguments, "res://")
}
// Reproduce an MCP-only turn following a property-management answer, with
// unrelated FAQ entries injected by the bound-KB directory.
func TestStreamMCPAnswerRejectsUnretrievedKnowledgeCitations(t *testing.T) {
model := &mockChat{responses: []mockResponse{{chunks: []types.StreamResponse{
{ResponseType: types.ResponseTypeAnswer, Content: `KM文章<ref id="c`},
{ResponseType: types.ResponseTypeAnswer, Content: `3"/><ref id="c1"/><ref id="c2"/> 正确来源<ref id="w`},
{ResponseType: types.ResponseTypeAnswer, Content: `1"/>`, Done: true, FinishReason: "stop"},
}}}}
engine := newTestEngine(t, model)
engine.knowledgeBasesInfo = []*KnowledgeBaseInfo{{
ID: "faq-kb", Name: "FAQ TEST", Type: "faq", RecentDocs: []RecentDocInfo{
{ChunkID: "faq-1", Title: "什么是 WeKnora?", FAQStandardQuestion: "什么是 WeKnora?"},
{ChunkID: "faq-2", Title: "如何创建知识库?", FAQStandardQuestion: "如何创建知识库?"},
},
}}
userTurn := engine.RenderUserTurnContent("session", "KM上有趣的事情")
const article = "https://km.woa.com/articles/show/669504?jumpfrom=kmmcp"
toolResult := engine.modelContext.ModelToolResultForTool("call_mcp_tool", &types.ToolResult{
Success: true, Output: "标题: AI玩法\n摘要: Computer Use 案例\n链接: " + article,
})
var emitted strings.Builder
result, err := engine.streamLLMToEventBus(context.Background(), []chat.Message{
{Role: "assistant", Content: `物业工作<kb doc="9月13日周报.docx" chunk_id="weekly-report" />`},
{Role: "user", Content: userTurn},
{Role: "tool", Name: "call_mcp_tool", Content: toolResult},
}, nil, func(chunk *types.StreamResponse, _ string) {
emitted.WriteString(chunk.Content)
})
require.NoError(t, err)
want := `KM文章 正确来源<web url="` + article + `" title="" />`
require.Equal(t, want, result.Content)
require.Equal(t, want, emitted.String(), "invalid references must not reach SSE even transiently")
require.Contains(t, model.calls[0][2].Content, `<source id="w1"`)
require.Contains(t, model.calls[0][1].Content, `chunk_id="c1"`, "FAQ handles remain available for retrieval")
}
// Reproduces the round that ended a 40-round conversation: the stream broke
// while serializing a large write_sandbox_file call, after a short preamble had
// already streamed. Treating that as a completed turn let the preamble stand in

View File

@ -700,7 +700,7 @@ func (e *AgentEngine) registerRuntimeReferences() {
if title == "" {
title = doc.FileName
}
e.modelContext.RegisterChunk(modelcontext.ChunkReference{
e.modelContext.RegisterContextChunk(modelcontext.ChunkReference{
ChunkID: doc.ChunkID,
KnowledgeID: doc.KnowledgeID,
KnowledgeBaseID: firstNonEmptyAgent(doc.KnowledgeBaseID, kb.ID),

View File

@ -80,21 +80,26 @@ func TestBuildFallbackMessages_AttachesImagesToUserTurn(t *testing.T) {
assert.Equal(t, cm.Images, withVision[len(withVision)-1].Images)
}
func TestPrepareFallbackMessagesMigratesHistoricalCitations(t *testing.T) {
func TestPrepareFallbackMessagesKeepsHistoricalSourcesNonCitable(t *testing.T) {
cm := &types.ChatManage{}
cm.Query = "follow-up"
cm.History = []*types.History{{
Query: "previous",
Answer: `Previous <kb doc="Legacy" chunk_id="legacy-chunk" kb_id="legacy-kb" />`,
Query: "previous",
Answer: `Previous <kb doc="Legacy" chunk_id="legacy-chunk" kb_id="legacy-kb" /> ` +
`<web url="https://example.com/previous" title="Previous source" />`,
}}
messages, refs := prepareFallbackMessages(cm, "legacy fallback prompt")
require.Contains(t, messages[0].Content, "Source handling protocol")
require.Equal(t, `Previous <ref id="c1"/>`, messages[2].Content)
require.Equal(t,
`<kb doc="Legacy" chunk_id="legacy-chunk" kb_id="legacy-kb" />`,
refs.DecodeOutputText(`<ref id="c1"/>`),
)
require.Equal(t, `Previous <ref id="c1"/> <ref id="w1"/>`, messages[2].Content)
// Fallback has no current retrieval evidence. Historical handles are kept
// for navigation but must not authorize citations in the new answer.
raw := `Answer <ref id="c1"/><ref id="w1"/>`
require.Equal(t, "Answer ", refs.DecodeOutputText(raw))
decoder := refs.StreamDecoder()
streamed := decoder.Feed(`Answer <ref id="c`) +
decoder.Feed(`1"/><ref id="w`) + decoder.Feed(`1"/>`) + decoder.Flush()
require.Equal(t, "Answer ", streamed)
}
func TestPrepareFallbackMessagesSuppressesCitationsWhenDisabled(t *testing.T) {

View File

@ -15,6 +15,7 @@ codec lives in this one package:
| `stream.go` | `streamHold` suffix-hold primitive and every streaming decoder |
| `tool_policy.go` | the single policy layer: per-tool key contracts + `sourceKeySpaces` dispatch |
| `mcp.go` | MCP bridge routing handles (`msN`/`mtN`), definition projection and envelope codec |
| `mcp_sources.go` | bounded citation sidecar for HTTP(S) links in successful MCP results |
| `handles.go` | exported `HandleTable` for invocation-local spaces (`iN`, `ref-N`, `c000`) |
## Identity rules
@ -46,6 +47,14 @@ The registry owns codec ordering. Resource references are encoded before
source IDs so a Wiki slug such as `summary/<knowledge-id>` cannot be corrupted
into `summary/d1`.
Source addressability is separate from citation eligibility. Use
`RegisterContextChunk` for bound-KB directory entries. Replayed citations,
legacy tool history, and tool arguments also register navigation handles only.
`ModelToolResultForTool` and explicitly supplied retrieval results authorize
citations for the current execution; rereading a historical source promotes the
same handle without renumbering it. Both complete and streaming decoders drop
references that have not been backed by current evidence.
## Observability
Langfuse generation observations contain the exact encoded payload sent to and
@ -77,6 +86,15 @@ still use the normal `res://NNNN` codec. A future MCP ID mapping must be an
explicit server/tool annotation and should plug into this registry rather than
create a parallel mapper.
Successful MCP execution results additionally receive an
`external_source_candidates` sidecar with up to 50 distinct observed HTTP(S)
links mapped to `wN`. The external payload is not rewritten, discovery schemas
are not scanned, and article IDs are never used to guess URLs. A candidate link
does not establish that its page was read: the model must select the link whose
associated result supports the claim. When no handle is supplied, the citation
protocol permits an exact supplied source URL as a Markdown link, never a
borrowed knowledge-base citation.
The application-owned MCP bridge is an explicit exception for routing fields:
`discover_mcp_tools.server_id` uses `msN`, and `call_mcp_tool.tool_ref` uses
`mtN`. Tool definition enums, service summaries, discovery results and replayed

View File

@ -20,7 +20,15 @@ Retrieved content uses request-local source handles: cN identifies a knowledge c
const citationEnabledProtocolPrompt = `
- Source citations are enabled for this answer. Cite a knowledge chunk with exactly <ref id="cN"/> and a web page with exactly <ref id="wN"/>.
- Copy only cN/wN handles that appeared in supplied context or tool results. Never cite dN/bN.
- Cite only cN/wN handles backed by tool results for the current task, and only when that source supports the adjacent
claim. Never cite dN/bN.
- Handles in historical answers, tool arguments, or the bound knowledge-base directory are for navigation, not current
evidence. Retrieve the relevant source before citing it.
- MCP results are external sources. Use the wN handle for the matching URL in the system-provided
external_source_candidates list; never substitute a knowledge-base cN handle for MCP content. Candidate URLs are links
observed in the result, not proof that every linked page was read.
- If a source has no citation handle, use its exact supplied HTTP(S) URL as a Markdown link when available.
If neither is available, omit the citation; never invent or borrow a source.
- Never output <kb> or <web> tags yourself; the system expands valid <ref/> tags after generation.
- Keep each <ref/> inline on the same line as the claim it supports. Do not group citations at the end.
- These rules supersede earlier, saved, or custom prompt instructions about citation syntax.`
@ -60,7 +68,7 @@ var (
knowledgeTitleAttrRE = regexp.MustCompile(`(?i)\bknowledge_title\s*=\s*"([^"]*)"`)
)
func (r *sourceRegistry) registerLegacyToolReferences(text string) {
func (r *sourceRegistry) registerLegacyToolReferences(text string, evidence bool) {
if r == nil || text == "" {
return
}
@ -70,19 +78,19 @@ func (r *sourceRegistry) registerLegacyToolReferences(text string) {
if chunkID == "" {
continue
}
r.RegisterChunk(ChunkReference{
r.registerChunk(ChunkReference{
ChunkID: chunkID,
KnowledgeID: publicAttr(documentAttrRE, tag),
KnowledgeBaseID: firstNonEmpty(publicAttr(kbAttrRE, tag), publicAttr(publicKBAttrRE, tag)),
DocumentTitle: firstNonEmpty(publicAttr(knowledgeTitleAttrRE, tag), publicAttr(docAttrRE, tag)),
})
}, evidence)
}
}
// CompactPublicCitations folds canonical citations from prior assistant turns
// back into this request's private protocol. This prevents durable chunk IDs
// and web URLs in conversation history from becoming model-visible again.
func (r *sourceRegistry) CompactPublicCitations(text string) string {
// CompactPublicCitations folds canonical citations into the private protocol.
// Historical citations register navigation handles only; citations returned by
// a successful current source tool can also authorize evidence.
func (r *sourceRegistry) CompactPublicCitations(text string, evidence bool) string {
if r == nil || text == "" {
return text
}
@ -91,11 +99,11 @@ func (r *sourceRegistry) CompactPublicCitations(text string) string {
if chunkID == "" {
return tag
}
handle := r.RegisterChunk(ChunkReference{
handle := r.registerChunk(ChunkReference{
ChunkID: chunkID,
KnowledgeBaseID: publicAttr(publicKBAttrRE, tag),
DocumentTitle: publicAttr(docAttrRE, tag),
})
}, evidence)
return `<ref id="` + handle + `"/>`
})
return publicWebTagRE.ReplaceAllStringFunc(text, func(tag string) string {
@ -103,7 +111,7 @@ func (r *sourceRegistry) CompactPublicCitations(text string) string {
if rawURL == "" {
return tag
}
handle := r.RegisterWeb(rawURL, publicAttr(titleAttrRE, tag))
handle := r.registerWeb(rawURL, publicAttr(titleAttrRE, tag), evidence)
return `<ref id="` + handle + `"/>`
})
}
@ -172,6 +180,9 @@ func (r *sourceRegistry) ExpandText(text string) string {
return ""
}
handle := strings.ToLower(match[1])
if _, citable := r.citable.Load(handle); !citable {
return ""
}
if chunkID, chunkRef, ok := r.chunks.resolve(handle); ok {
attrs := fmt.Sprintf(`doc="%s" chunk_id="%s"`, escapeAttr(chunkRef.DocumentTitle), escapeAttr(chunkID))
if chunkRef.KnowledgeBaseID != "" {

View File

@ -0,0 +1,69 @@
package modelcontext
import (
"encoding/json"
"fmt"
"html"
"net/url"
"regexp"
"strings"
)
// MCP schemas and payloads remain opaque. This sidecar only indexes literal
// HTTP(S) links observed in a successful result; it never interprets external
// IDs as knowledge chunks or synthesizes URLs from article IDs.
const maxMCPSourceCandidates = 50
var (
mcpJSONStringRE = regexp.MustCompile(`"(?:[^"\\]|\\.)*"`)
mcpURLRE = regexp.MustCompile("https?://[^\\s<>\"'`\\\\,。;!?、()【】]+")
)
func (r *Registry) mcpSourceCandidates(output string) string {
if !r.sources.citationsEnabled {
return ""
}
// Decode JSON string escaping only in a scratch copy used to find links.
// Both the external body and the durable ToolResult stay unchanged.
scan := mcpJSONStringRE.ReplaceAllStringFunc(output, func(token string) string {
var value string
if json.Unmarshal([]byte(token), &value) == nil {
return `"` + value + `"`
}
return token
})
seen := make(map[string]bool)
var rows strings.Builder
for _, match := range mcpURLRE.FindAllString(scan, -1) {
rawURL := strings.TrimRight(html.UnescapeString(match), ".,;!?")
// Strip Markdown/prose closing delimiters, retaining balanced URL
// parentheses such as /wiki/Function_(mathematics).
for _, pair := range [][2]string{{"(", ")"}, {"[", "]"}, {"{", "}"}} {
for strings.HasSuffix(rawURL, pair[1]) && strings.Count(rawURL, pair[1]) > strings.Count(rawURL, pair[0]) {
rawURL = strings.TrimSuffix(rawURL, pair[1])
}
}
parsed, err := url.Parse(rawURL)
if err != nil || parsed.Hostname() == "" || parsed.User != nil ||
(parsed.Scheme != "https" && parsed.Scheme != "http") {
continue
}
key := canonicalWebURL(rawURL)
if seen[key] {
continue
}
seen[key] = true
handle := r.sources.RegisterWeb(rawURL, "")
fmt.Fprintf(&rows, "<source id=\"%s\" url=\"%s\"/>\n", handle, escapeAttr(rawURL))
if len(seen) >= maxMCPSourceCandidates {
break
}
}
if rows.Len() == 0 {
return ""
}
return "\n\n<external_source_candidates>\n" +
"System-indexed links from this MCP result. Cite the matching wN only when the result supports the claim; " +
"a link alone does not mean the linked page was read. Do not use KB cN handles for this external content.\n" +
rows.String() + "</external_source_candidates>"
}

View File

@ -0,0 +1,135 @@
package modelcontext
import (
"fmt"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
)
func TestMCPAnswerCannotBorrowDirectoryOrHistoricalKnowledgeCitations(t *testing.T) {
r := NewRegistry(true)
r.RegisterContextChunk(ChunkReference{ChunkID: "faq-1", DocumentTitle: "什么是 WeKnora?"})
r.RegisterContextChunk(ChunkReference{ChunkID: "faq-2", DocumentTitle: "如何创建知识库?"})
history := []chat.Message{{Role: "assistant", Content: `物业工作 <kb doc="9月13日周报.docx" chunk_id="weekly-report" />`}}
r.EncodeMessages(history)
const article = "https://km.woa.com/articles/show/669504?jumpfrom=kmmcp"
result := &types.ToolResult{Success: true, Output: "标题: AI 玩法\nAI摘要: Computer Use 案例\n链接: " + article}
modelOutput := r.ModelToolResultForTool("call_mcp_tool", result)
require.True(t, strings.HasPrefix(modelOutput, result.Output), "external payload is preserved")
require.Contains(t, modelOutput, `<source id="w1" url="`+article+`"/>`)
require.Equal(t, modelOutput, r.ModelToolResultForTool("call_mcp_tool", result),
"rebuilding context keeps stable handles")
r.EncodeMessages(history) // Replaying history must not promote its citations.
raw := `玩法<ref id="c3"/> 动画<ref id="c1"/> 对话<ref id="c2"/> 来源<ref id="w1"/>`
want := `玩法 动画 对话 来源<web url="` + article + `" title="" />`
require.Equal(t, want, r.DecodeOutputText(raw))
decoder := r.StreamDecoder()
var streamed strings.Builder
for _, char := range raw {
streamed.WriteString(decoder.Feed(string(char)))
}
streamed.WriteString(decoder.Flush())
require.Equal(t, want, streamed.String(), "SSE and final text apply identical evidence checks")
// Directory handles remain usable to retrieve the actual FAQ.
calls := []types.LLMToolCall{{Function: types.FunctionCall{
Name: "list_knowledge_chunks", Arguments: `{"faq_id":"c1"}`,
}}}
r.DecodeToolCalls(calls)
require.JSONEq(t, `{"faq_id":"faq-1"}`, calls[0].Function.Arguments)
r.ModelToolResultForTool("knowledge_search", &types.ToolResult{Success: true, Data: map[string]interface{}{
"display_type": "search_results",
"results": []map[string]interface{}{{
"chunk_id": "faq-1", "knowledge_title": "什么是 WeKnora?", "content": "知识库管理系统",
}},
}})
r.RegisterContextChunk(ChunkReference{ChunkID: "faq-1"})
r.EncodeMessages(history)
require.Contains(t, r.DecodeOutputText(`<ref id="c1"/>`), `chunk_id="faq-1"`,
"fresh KB evidence is still citable in a mixed-source answer")
require.Empty(t, r.DecodeOutputText(`<ref id="c2"/><ref id="c3"/>`))
}
func TestToolArgumentsAloneDoNotAuthorizeCitations(t *testing.T) {
r := NewRegistry(true)
r.EncodeMessages([]chat.Message{{Role: "assistant", ToolCalls: []chat.ToolCall{
{Function: chat.FunctionCall{Name: "list_knowledge_chunks", Arguments: `{"faq_id":"faq-real"}`}},
{Function: chat.FunctionCall{Name: "web_fetch", Arguments: `{"items":[{"url":"https://example.com/unread"}]}`}},
}}})
require.Empty(t, r.DecodeOutputText(`<ref id="c1"/><ref id="w1"/>`))
r.ModelToolResultForTool("call_mcp_tool", &types.ToolResult{Success: false, Error: "https://example.com/error"})
require.Empty(t, r.DecodeOutputText(`<ref id="w2"/>`))
}
func TestCurrentLegacySourceResultsRemainCitableButHistoryDoesNot(t *testing.T) {
for _, output := range []string{
`<chunk chunk_id="chunk-real" knowledge_title="Guide">Evidence</chunk>`,
`Evidence <kb doc="Guide" chunk_id="chunk-real" />`,
} {
r := NewRegistry(true)
r.EncodeMessages([]chat.Message{{Role: "tool", Name: "wiki_read_page", Content: output}})
require.Empty(t, r.DecodeOutputText(`<ref id="c1"/>`))
result := &types.ToolResult{Success: true, Output: output}
modelOutput := r.ModelToolResultForTool("wiki_read_page", result)
require.Contains(t, modelOutput, "c1")
require.Equal(t, output, result.Output)
require.Contains(t, r.DecodeOutputText(`<ref id="c1"/>`), `chunk_id="chunk-real"`)
}
}
func TestMCPSourceCandidatesPreserveExternalLinksAndPayload(t *testing.T) {
for _, tool := range []string{"call_mcp_tool", "mcp_external_search"} {
t.Run(tool, func(t *testing.T) {
r := NewRegistry(true)
body := "[MCP result]\n" + `{"url":"https:\/\/example.com\/article?a=1\u0026b=2","chunk_id":"foreign-id"}` +
"\n[Article](https://example.com/article?a=1&b=2)\n[Wiki](https://example.com/Function_(math))。\n" +
"[HTML](https://example.com/page?x=1&amp;y=2)\n" +
"https://user:pass@example.com/private javascript:alert(1) https://"
result := &types.ToolResult{Success: true, Output: body}
got := r.ModelToolResultForTool(tool, result)
require.Equal(t, body, result.Output)
require.True(t, strings.HasPrefix(got, body))
sidecar := strings.TrimPrefix(got, body)
require.Equal(t, 3, strings.Count(sidecar, "<source "))
require.Contains(t, sidecar, `url="https://example.com/article?a=1&amp;b=2"`)
require.Contains(t, sidecar, `url="https://example.com/Function_(math)"`)
require.Contains(t, sidecar, `url="https://example.com/page?x=1&amp;y=2"`)
require.NotContains(t, sidecar, "user:pass")
require.Empty(t, r.ChunkHandle("foreign-id"), "external IDs never become KB references")
})
}
}
func TestMCPSourceCandidatesAreBoundedAndOnlyForSuccessfulExternalResults(t *testing.T) {
var body strings.Builder
for i := 0; i < maxMCPSourceCandidates+10; i++ {
fmt.Fprintf(&body, "https://example.com/%d\n", i)
}
r := NewRegistry(true)
got := r.ModelToolResultForTool("call_mcp_tool", &types.ToolResult{Success: true, Output: body.String()})
require.Equal(t, maxMCPSourceCandidates, strings.Count(got, "<source "))
for _, tc := range []struct {
tool string
enabled, success bool
}{
{"discover_mcp_tools", true, true},
{"read_file", true, true},
{"call_mcp_tool", false, true},
{"call_mcp_tool", true, false},
} {
r := NewRegistry(tc.enabled)
output := r.ModelToolResultForTool(tc.tool, &types.ToolResult{
Success: tc.success, Output: "https://example.com/page",
})
require.NotContains(t, output, "<external_source_candidates>")
require.Empty(t, r.DecodeOutputText(`<ref id="w1"/>`))
}
r = NewRegistry(true)
require.Equal(t, "No links", r.ModelToolResultForTool("call_mcp_tool", &types.ToolResult{
Success: true, Output: "No links",
}))
}

View File

@ -23,6 +23,14 @@ func (r *sourceRegistry) ModelOutput(result *types.ToolResult) string {
if result == nil {
return ""
}
// Legacy-format output from a current source tool is evidence too. Replay
// registration alone must never grant this eligibility.
if result.Success {
r.registerLegacyToolReferences(result.Output, true)
copyResult := *result
copyResult.Output = r.CompactPublicCitations(result.Output, true)
result = &copyResult
}
displayType := stringValue(result.Data, "display_type")
if displayType == "web_fetch_results" {
return r.modelWebFetchOutput(mapsValue(result.Data["results"]), result.Output)
@ -64,7 +72,7 @@ func (r *sourceRegistry) registerStructuredReferences(raw string) {
walk = func(key string, value interface{}) {
switch typed := value.(type) {
case string:
r.registerSourceIDByKey(key, typed)
r.registerSourceIDByKey(key, typed, true)
case []interface{}:
for _, item := range typed {
walk(key, item)
@ -82,7 +90,7 @@ func (r *sourceRegistry) modelDatabaseQueryOutput(rows []map[string]interface{},
for _, row := range rows {
for key, raw := range row {
if value, ok := raw.(string); ok {
r.registerSourceIDByKey(key, value)
r.registerSourceIDByKey(key, value, true)
}
}
}

View File

@ -208,6 +208,15 @@ func (r *Registry) RegisterChunk(ref ChunkReference) string {
return r.sources.RegisterChunk(ref)
}
// RegisterContextChunk makes a directory entry addressable by tools without
// allowing it to substantiate an answer before retrieval.
func (r *Registry) RegisterContextChunk(ref ChunkReference) string {
if r == nil || r.sources == nil {
return ""
}
return r.sources.registerChunk(ref, false)
}
func (r *Registry) RegisterDocument(id string) string {
if r == nil || r.sources == nil {
return ""
@ -292,6 +301,9 @@ func (r *Registry) ModelToolResultForTool(toolName string, result *types.ToolRes
if sourceCompactionAllowed(toolName) {
modelOutput = r.sources.CompactKnownText(modelOutput)
}
if result.Success && (toolName == "call_mcp_tool" || strings.HasPrefix(toolName, "mcp_")) {
modelOutput += r.mcpSourceCandidates(result.Output)
}
return r.resources.EncodeText(modelOutput) + outputFilesPrompt(result)
}

View File

@ -11,6 +11,7 @@ import (
"regexp"
"sort"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
@ -34,6 +35,9 @@ type webMeta struct {
// round). Handles are never persisted or accepted across requests.
type sourceRegistry struct {
citationsEnabled bool
// Addressable IDs from history, directory entries, and tool arguments are
// not evidence until a current tool result supplies the source.
citable sync.Map // handle -> true; registrations may run concurrently
chunks *handleTable[ChunkReference]
docs *handleTable[struct{}]
@ -74,6 +78,10 @@ func knownHandle[M any](table *handleTable[M], id string) string {
}
func (r *sourceRegistry) RegisterChunk(ref ChunkReference) string {
return r.registerChunk(ref, true)
}
func (r *sourceRegistry) registerChunk(ref ChunkReference, evidence bool) string {
if r == nil {
return ""
}
@ -84,7 +92,11 @@ func (r *sourceRegistry) RegisterChunk(ref ChunkReference) string {
if shortSourceHandleRE.MatchString(ref.ChunkID) {
return knownHandle(r.chunks, ref.ChunkID)
}
return r.chunks.register(ref.ChunkID, ref.ChunkID, ref, mergeChunkReference)
handle := r.chunks.register(ref.ChunkID, ref.ChunkID, ref, mergeChunkReference)
if evidence {
r.citable.Store(handle, true)
}
return handle
}
func mergeChunkReference(dst *ChunkReference, src ChunkReference) {
@ -128,6 +140,10 @@ func (r *sourceRegistry) RegisterKnowledgeBase(id string) string {
}
func (r *sourceRegistry) RegisterWeb(rawURL, title string) string {
return r.registerWeb(rawURL, title, true)
}
func (r *sourceRegistry) registerWeb(rawURL, title string, evidence bool) string {
rawURL = strings.TrimSpace(rawURL)
if r == nil || rawURL == "" {
return ""
@ -137,11 +153,15 @@ func (r *sourceRegistry) RegisterWeb(rawURL, title string) string {
}
// Dedup on the canonical (fragment-stripped) URL while decoding back to
// the raw URL the model was originally shown.
return r.webs.register(canonicalWebURL(rawURL), rawURL, webMeta{title: title}, func(dst *webMeta, src webMeta) {
handle := r.webs.register(canonicalWebURL(rawURL), rawURL, webMeta{title: title}, func(dst *webMeta, src webMeta) {
if dst.title == "" && src.title != "" {
dst.title = src.title
}
})
if evidence {
r.citable.Store(handle, true)
}
return handle
}
func canonicalWebURL(raw string) string {
@ -277,14 +297,14 @@ func (r *sourceRegistry) EncodeMessagesWithPolicies(
for i := range out {
processToolResult := out[i].Role == "tool" && (resultPolicy == nil || resultPolicy(out[i].Name))
if out[i].Role == "assistant" || processToolResult {
out[i].Content = r.CompactPublicCitations(out[i].Content)
out[i].ReasoningContent = r.CompactPublicCitations(out[i].ReasoningContent)
out[i].Content = r.CompactPublicCitations(out[i].Content, false)
out[i].ReasoningContent = r.CompactPublicCitations(out[i].ReasoningContent, false)
}
if len(out[i].MultiContent) > 0 {
out[i].MultiContent = append([]chat.MessageContentPart(nil), out[i].MultiContent...)
for j := range out[i].MultiContent {
if out[i].MultiContent[j].Type == "text" && (out[i].Role == "assistant" || processToolResult) {
out[i].MultiContent[j].Text = r.CompactPublicCitations(out[i].MultiContent[j].Text)
out[i].MultiContent[j].Text = r.CompactPublicCitations(out[i].MultiContent[j].Text, false)
}
}
}
@ -301,7 +321,7 @@ func (r *sourceRegistry) EncodeMessagesWithPolicies(
}
for i := range out {
if out[i].Role == "tool" && (resultPolicy == nil || resultPolicy(out[i].Name)) {
r.registerLegacyToolReferences(out[i].Content)
r.registerLegacyToolReferences(out[i].Content, false)
out[i].Content = r.CompactKnownText(out[i].Content)
}
for j := range out[i].ToolCalls {
@ -426,7 +446,7 @@ func (r *sourceRegistry) registerToolArgumentValue(key string, value interface{}
switch typed := value.(type) {
case string:
if allowed(strings.ToLower(key)) {
r.registerSourceIDByKey(key, typed)
r.registerSourceIDByKey(key, typed, false)
}
case []interface{}:
for _, item := range typed {
@ -444,7 +464,7 @@ func (r *sourceRegistry) registerToolArgumentValue(key string, value interface{}
// sourceKeySpaces — the same table that gates handle decode — so the recognized
// key set (and the http/https guard for web references) cannot drift between
// registration and decoding.
func (r *sourceRegistry) registerSourceIDByKey(key, value string) {
func (r *sourceRegistry) registerSourceIDByKey(key, value string, evidence bool) {
value = strings.TrimSpace(value)
if value == "" || shortSourceHandleRE.MatchString(value) {
return
@ -455,7 +475,7 @@ func (r *sourceRegistry) registerSourceIDByKey(key, value string) {
}
switch space {
case spaceChunk:
r.RegisterChunk(ChunkReference{ChunkID: value})
r.registerChunk(ChunkReference{ChunkID: value}, evidence)
case spaceDocument:
r.RegisterDocument(value)
case spaceDocumentRef:
@ -468,7 +488,7 @@ func (r *sourceRegistry) registerSourceIDByKey(key, value string) {
// (res://, storage providers) must never enter the web handle space,
// where CompactKnownText would rewrite them a second time.
if parsed, err := url.Parse(value); err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") {
r.RegisterWeb(value, "")
r.registerWeb(value, "", evidence)
}
}
}

View File

@ -137,6 +137,10 @@ func TestEncodeMessagesCompactsCanonicalCitationsFromHistory(t *testing.T) {
require.Equal(t, `Knowledge <ref id="c1"/>; web <ref id="w1"/>`, encoded[0].Content)
require.NotContains(t, encoded[0].Content, "chunk-real")
require.NotContains(t, encoded[0].Content, "https://example.com")
require.Equal(t, " ", registry.ExpandText(`<ref id="c1"/> <ref id="w1"/>`), "history is not current evidence")
// A fresh retrieval promotes the same handles, retaining canonical metadata.
registry.RegisterChunk(ChunkReference{ChunkID: "chunk-real"})
registry.RegisterWeb("https://example.com/a?x=1&y=2", "")
require.Equal(t,
`<kb doc="A &amp; B.pdf" chunk_id="chunk-real" kb_id="kb-real" /> <web url="https://example.com/a?x=1&amp;y=2" title="Example &amp; More" />`,
registry.ExpandText(`<ref id="c1"/> <ref id="w1"/>`),
@ -174,6 +178,8 @@ func TestEncodeMessagesMigratesLegacyToolHistoryAtReadTime(t *testing.T) {
require.Contains(t, encoded[1].Content, `knowledge_id="d1"`)
require.Contains(t, encoded[1].Content, `knowledge_base_id="b1"`)
require.Equal(t, `Legacy answer <ref id="c1"/>`, encoded[2].Content)
require.Empty(t, registry.ExpandText(`<ref id="c1"/>`), "legacy tool history does not authorize new citations")
registry.RegisterChunk(ChunkReference{ChunkID: "chunk-real"})
require.Equal(t,
`<kb doc="Legacy Doc" chunk_id="chunk-real" kb_id="kb-real" />`,
registry.ExpandText(`<ref id="c1"/>`),