refactor(modelcontext): unify source-key gating behind one dispatch table

sourceKeySpaces becomes the single table of ID-bearing keys: it drives
handle registration (registerSourceIDByKey) and the alias-decode gate
(walkJSON / unresolved-handle audit), replacing the separate decodableKeys
list that had to be kept in sync by hand. The per-tool sourceIDKeys policy
remains the outer gate; a new test asserts every policy key has a
registration space.

The generic nil-policy wrappers (sourceRegistry.DecodeToolCalls,
EncodeMessages) are removed; the two tests that exercised anonymous-tool
decoding now go through the public Registry with real tool names.
main
wizardchen 2026-07-27 01:08:11 +08:00 committed by lyingbug
parent 3b9de5de8e
commit 286eadac0a
4 changed files with 93 additions and 58 deletions

View File

@ -258,16 +258,10 @@ func (r *sourceRegistry) ChunkAlias(id string) string {
return ""
}
// DecodeToolCalls restores exact alias-valued JSON strings before tools parse
// or validate their arguments. It never performs substring replacement.
func (r *sourceRegistry) DecodeToolCalls(toolCalls []types.LLMToolCall) {
r.DecodeToolCallsWithPolicy(toolCalls, nil)
}
// toolArgumentPolicy decides whether a source-bearing JSON key belongs to a
// particular tool contract. A nil policy retains the codec's legacy generic
// behavior; application request lifecycles should provide an explicit policy
// through Registry.
// particular tool contract. Request lifecycles always pass the per-tool policy
// (sourceArgumentAllowed); a nil policy allows every key and exists only for
// package-internal replay paths that predate per-tool contracts.
type toolArgumentPolicy func(toolName, key string) bool
// DecodeToolCallsWithPolicy restores aliases only for fields explicitly owned
@ -319,7 +313,7 @@ func (r *sourceRegistry) collectUnresolvedToolHandles(
switch typed := value.(type) {
case string:
key = strings.ToLower(key)
if _, ok := decodableKeys[key]; !ok || !allowed(key) {
if _, ok := sourceKeySpaces[key]; !ok || !allowed(key) {
return
}
handle := strings.TrimSpace(typed)
@ -337,13 +331,9 @@ func (r *sourceRegistry) collectUnresolvedToolHandles(
}
}
// EncodeMessages compacts known real identifiers in assistant tool-call replay.
func (r *sourceRegistry) EncodeMessages(messages []chat.Message) []chat.Message {
return r.EncodeMessagesWithPolicies(messages, nil, nil)
}
// EncodeMessagesWithPolicies additionally gates source processing for tool
// result messages. A nil result policy retains the legacy generic behavior.
// EncodeMessagesWithPolicies compacts known real identifiers in replayed
// messages and gates source processing for tool results by tool name. A nil
// policy retains the legacy generic behavior for package-internal callers.
func (r *sourceRegistry) EncodeMessagesWithPolicies(
messages []chat.Message,
argumentPolicy toolArgumentPolicy,
@ -523,27 +513,32 @@ func (r *sourceRegistry) registerToolArgumentValue(key string, value interface{}
}
// registerSourceIDByKey is the single key→source-space dispatch used for tool
// arguments, structured tool results, and database rows. Keeping one dispatch
// stops the recognized key set (and the http/https guard for web references)
// from drifting between call sites; it must stay in sync with decodableKeys.
// arguments, structured tool results, and database rows. It is driven by
// sourceKeySpaces — the same table that gates alias decode — so the recognized
// key set (and the http/https guard for web references) cannot drift between
// registration and decoding.
func (r *sourceRegistry) registerSourceIDByKey(key, value string) {
value = strings.TrimSpace(value)
if value == "" || shortSourceAliasRE.MatchString(value) {
return
}
switch strings.ToLower(key) {
case "chunk_id", "faq_id", "chunk_ids", "faq_ids":
space, ok := sourceKeySpaces[strings.ToLower(key)]
if !ok {
return
}
switch space {
case spaceChunk:
r.RegisterChunk(ChunkReference{ChunkID: value})
case "knowledge_id", "knowledge_ids", "suspected_knowledge_ids":
case spaceDocument:
r.RegisterDocument(value)
case "source_refs":
case spaceDocumentRef:
// Stored refs use "knowledgeID|title"; only the ID part is durable.
r.RegisterDocument(strings.TrimSpace(strings.SplitN(value, "|", 2)[0]))
case "knowledge_base", "knowledge_base_id", "knowledge_base_ids", "kb_id", "kb_ids":
case spaceKnowledgeBase:
r.RegisterKnowledgeBase(value)
case "url", "urls":
case spaceWeb:
// Only public web pages become web references. Internal schemes
// (res://, storage providers) must never enter the web alias space,
// (res://, storage providers) must never enter the web handle space,
// where CompactKnownText would rewrite them a second time.
if parsed, err := url.Parse(value); err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") {
r.RegisterWeb(value, "")
@ -567,18 +562,6 @@ func (r *sourceRegistry) decodeJSONWithPolicy(raw string, encode bool, allowed f
return string(encoded)
}
// decodableKeys mirrors the ID-bearing keys recognized by
// registerToolArgumentValue. Alias -> real substitution on decode is restricted
// to these keys so that free-text values (e.g. a grep/search query that happens
// to equal "d1") are never rewritten into internal identifiers.
var decodableKeys = map[string]struct{}{
"chunk_id": {}, "faq_id": {}, "chunk_ids": {}, "faq_ids": {},
"knowledge_id": {}, "knowledge_ids": {}, "suspected_knowledge_ids": {}, "source_refs": {},
"knowledge_base": {}, "knowledge_base_id": {}, "knowledge_base_ids": {}, "kb_id": {}, "kb_ids": {},
"url": {}, "urls": {},
}
func (r *sourceRegistry) walkJSON(key string, value interface{}, encode bool, allowed func(string) bool) interface{} {
switch typed := value.(type) {
case string:
@ -595,7 +578,7 @@ func (r *sourceRegistry) walkJSON(key string, value interface{}, encode bool, al
}
// Decode only ID-bearing keys, and only when the value is alias-shaped,
// so ordinary strings that coincidentally equal an alias are preserved.
if _, ok := decodableKeys[strings.ToLower(key)]; !ok {
if _, ok := sourceKeySpaces[strings.ToLower(key)]; !ok {
return typed
}
if !shortSourceAliasRE.MatchString(strings.TrimSpace(typed)) {

View File

@ -67,41 +67,42 @@ func TestRegistrySuppressesSourceCitationsWhenDisabled(t *testing.T) {
}
func TestRegistryDecodesAliasesInNestedToolArguments(t *testing.T) {
registry := newSourceRegistry()
registry := NewRegistry(true)
registry.RegisterDocument("knowledge-uuid-1")
registry.RegisterKnowledgeBase("kb-uuid-1")
registry.RegisterWeb("https://example.com/page", "Example")
registry.RegisterChunk(ChunkReference{ChunkID: "chunk-uuid-1"})
// Handles nested inside arrays and objects must decode as long as the key
// belongs to the named tool's declared source contract.
calls := []types.LLMToolCall{{
Function: types.FunctionCall{
Arguments: `{"knowledge_id":"d1","knowledge_base_ids":["b1"],"url":"w1","chunk_id":"c1"}`,
Name: "list_knowledge_chunks",
Arguments: `{"knowledge_id":"d1","filters":{"chunk_id":"c1","faq_id":["c1"]}}`,
},
}}
registry.DecodeToolCalls(calls)
require.JSONEq(t,
`{"knowledge_id":"knowledge-uuid-1","knowledge_base_ids":["kb-uuid-1"],"url":"https://example.com/page","chunk_id":"chunk-uuid-1"}`,
`{"knowledge_id":"knowledge-uuid-1","filters":{"chunk_id":"chunk-uuid-1","faq_id":["chunk-uuid-1"]}}`,
calls[0].Function.Arguments,
)
}
func TestDecodeToolCallsOnlyRewritesAliasBearingKeys(t *testing.T) {
registry := newSourceRegistry()
registry.RegisterDocument("knowledge-uuid-1")
registry.RegisterChunk(ChunkReference{ChunkID: "chunk-uuid-1"})
registry := NewRegistry(true)
registry.RegisterKnowledgeBase("kb-uuid-1")
// A free-text field (query) whose value coincidentally equals an alias must
// be preserved verbatim, while ID-bearing keys still resolve to real IDs.
// A free-text field (query) whose value coincidentally equals a handle must
// be preserved verbatim, while declared ID-bearing keys resolve to real IDs.
calls := []types.LLMToolCall{{
Function: types.FunctionCall{
Arguments: `{"query":"d1","knowledge_id":"d1","content":"see c1 for details","chunk_id":"c1"}`,
Name: "knowledge_search",
Arguments: `{"query":"b1","content":"see b1 for details","knowledge_base_ids":["b1"]}`,
},
}}
registry.DecodeToolCalls(calls)
require.JSONEq(t,
`{"query":"d1","knowledge_id":"knowledge-uuid-1","content":"see c1 for details","chunk_id":"chunk-uuid-1"}`,
`{"query":"b1","content":"see b1 for details","knowledge_base_ids":["kb-uuid-1"]}`,
calls[0].Function.Arguments,
)
}
@ -131,7 +132,7 @@ func TestEncodeMessagesCompactsCanonicalCitationsFromHistory(t *testing.T) {
`web <web url="https://example.com/a?x=1&amp;y=2" title="Example &amp; More" />`,
}}
encoded := registry.EncodeMessages(messages)
encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil)
require.Equal(t, `Knowledge <ref id="c1"/>; web <ref id="w1"/>`, encoded[0].Content)
require.NotContains(t, encoded[0].Content, "chunk-real")
require.NotContains(t, encoded[0].Content, "https://example.com")
@ -165,7 +166,7 @@ func TestEncodeMessagesMigratesLegacyToolHistoryAtReadTime(t *testing.T) {
},
}
encoded := registry.EncodeMessages(messages)
encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil)
require.JSONEq(t, `{"knowledge_base_ids":["b1"],"knowledge_ids":["d1"]}`,
encoded[0].ToolCalls[0].Function.Arguments)
require.Contains(t, encoded[1].Content, `chunk_id="c1"`)
@ -185,7 +186,7 @@ func TestEncodeMessagesDoesNotTreatLegacyPromptExampleAsARealSource(t *testing.T
Content: `Old rule: cite <kb doc="..." chunk_id="..." />`,
}}
encoded := registry.EncodeMessages(messages)
encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil)
require.Equal(t, messages[0].Content, encoded[0].Content)
require.Zero(t, registry.Count())
}
@ -303,9 +304,13 @@ func TestModelOutputCompactsLabeledWikiReferences(t *testing.T) {
require.Contains(t, output, `<knowledge_base_id>b1</knowledge_base_id>`)
require.Contains(t, output, `knowledge_id="d1"`)
toolCalls := []types.LLMToolCall{{Function: types.FunctionCall{Arguments: `{"knowledge_id":"d1","knowledge_base_id":"b1"}`}}}
registry.DecodeToolCalls(toolCalls)
require.JSONEq(t, `{"knowledge_id":"doc-real-id","knowledge_base_id":"kb-real-id"}`, toolCalls[0].Function.Arguments)
toolCalls := []types.LLMToolCall{
{Function: types.FunctionCall{Name: "wiki_read_source_doc", Arguments: `{"knowledge_id":"d1"}`}},
{Function: types.FunctionCall{Name: "wiki_search", Arguments: `{"knowledge_base_id":"b1"}`}},
}
registry.DecodeToolCallsWithPolicy(toolCalls, sourceArgumentAllowed)
require.JSONEq(t, `{"knowledge_id":"doc-real-id"}`, toolCalls[0].Function.Arguments)
require.JSONEq(t, `{"knowledge_base_id":"kb-real-id"}`, toolCalls[1].Function.Arguments)
}
func TestModelOutputGraphResultsUseChunkAliases(t *testing.T) {

View File

@ -18,6 +18,32 @@ const (
var issueHandleShapeRE = regexp.MustCompile(`^i[1-9][0-9]*$`)
// sourceKeySpace identifies which source-handle space an ID-bearing JSON key
// belongs to.
type sourceKeySpace int
const (
spaceChunk sourceKeySpace = iota
spaceDocument
spaceDocumentRef // "knowledgeID|title" stored refs; only the ID is durable
spaceKnowledgeBase
spaceWeb
)
// sourceKeySpaces is the single table of ID-bearing keys the source codec
// understands. It drives both handle registration (registerSourceIDByKey) and
// the alias-shaped-value decode gate (walkJSON), so registration and decoding
// cannot drift apart. Every key referenced by a toolHandlePolicies sourceIDKeys
// set must appear here (enforced by a test).
var sourceKeySpaces = map[string]sourceKeySpace{
"chunk_id": spaceChunk, "faq_id": spaceChunk, "chunk_ids": spaceChunk, "faq_ids": spaceChunk,
"knowledge_id": spaceDocument, "knowledge_ids": spaceDocument, "suspected_knowledge_ids": spaceDocument,
"source_refs": spaceDocumentRef,
"knowledge_base": spaceKnowledgeBase, "knowledge_base_id": spaceKnowledgeBase,
"knowledge_base_ids": spaceKnowledgeBase, "kb_id": spaceKnowledgeBase, "kb_ids": spaceKnowledgeBase,
"url": spaceWeb, "urls": spaceWeb,
}
type toolHandlePolicy struct {
sourceIDKeys map[string]struct{}
sourceTextKeys map[string]struct{}

View File

@ -0,0 +1,21 @@
package modelcontext
import "testing"
// Every key a tool declares as a source-ID argument must have a registration
// space, otherwise its values could be decoded from handles but never
// registered (or vice versa) — the drift this table exists to prevent.
func TestEverySourceIDKeyHasARegistrationSpace(t *testing.T) {
for tool, policy := range toolHandlePolicies {
for key := range policy.sourceIDKeys {
if _, ok := sourceKeySpaces[key]; !ok {
t.Errorf("tool %s declares source key %q with no entry in sourceKeySpaces", tool, key)
}
}
for key := range policy.sourceTextKeys {
if _, ok := sourceKeySpaces[key]; ok {
t.Errorf("tool %s declares %q as free text but sourceKeySpaces also claims it as an ID key", tool, key)
}
}
}
}