From 286eadac0afd7d65d8d8d692e98f49b4c44fbf3b Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 27 Jul 2026 01:08:11 +0800 Subject: [PATCH] refactor(modelcontext): unify source-key gating behind one dispatch table sourceKeySpaces becomes the single table of ID-bearing keys: it drives handle registration (registerSourceIDByKey) and the alias-decode gate (walkJSON / unresolved-handle audit), replacing the separate decodableKeys list that had to be kept in sync by hand. The per-tool sourceIDKeys policy remains the outer gate; a new test asserts every policy key has a registration space. The generic nil-policy wrappers (sourceRegistry.DecodeToolCalls, EncodeMessages) are removed; the two tests that exercised anonymous-tool decoding now go through the public Registry with real tool names. --- internal/modelcontext/sources.go | 63 +++++++++-------------- internal/modelcontext/sources_test.go | 41 ++++++++------- internal/modelcontext/tool_policy.go | 26 ++++++++++ internal/modelcontext/tool_policy_test.go | 21 ++++++++ 4 files changed, 93 insertions(+), 58 deletions(-) create mode 100644 internal/modelcontext/tool_policy_test.go diff --git a/internal/modelcontext/sources.go b/internal/modelcontext/sources.go index aee3ed52..2f0369f5 100644 --- a/internal/modelcontext/sources.go +++ b/internal/modelcontext/sources.go @@ -258,16 +258,10 @@ func (r *sourceRegistry) ChunkAlias(id string) string { return "" } -// DecodeToolCalls restores exact alias-valued JSON strings before tools parse -// or validate their arguments. It never performs substring replacement. -func (r *sourceRegistry) DecodeToolCalls(toolCalls []types.LLMToolCall) { - r.DecodeToolCallsWithPolicy(toolCalls, nil) -} - // toolArgumentPolicy decides whether a source-bearing JSON key belongs to a -// particular tool contract. A nil policy retains the codec's legacy generic -// behavior; application request lifecycles should provide an explicit policy -// through Registry. +// particular tool contract. Request lifecycles always pass the per-tool policy +// (sourceArgumentAllowed); a nil policy allows every key and exists only for +// package-internal replay paths that predate per-tool contracts. type toolArgumentPolicy func(toolName, key string) bool // DecodeToolCallsWithPolicy restores aliases only for fields explicitly owned @@ -319,7 +313,7 @@ func (r *sourceRegistry) collectUnresolvedToolHandles( switch typed := value.(type) { case string: key = strings.ToLower(key) - if _, ok := decodableKeys[key]; !ok || !allowed(key) { + if _, ok := sourceKeySpaces[key]; !ok || !allowed(key) { return } handle := strings.TrimSpace(typed) @@ -337,13 +331,9 @@ func (r *sourceRegistry) collectUnresolvedToolHandles( } } -// EncodeMessages compacts known real identifiers in assistant tool-call replay. -func (r *sourceRegistry) EncodeMessages(messages []chat.Message) []chat.Message { - return r.EncodeMessagesWithPolicies(messages, nil, nil) -} - -// EncodeMessagesWithPolicies additionally gates source processing for tool -// result messages. A nil result policy retains the legacy generic behavior. +// EncodeMessagesWithPolicies compacts known real identifiers in replayed +// messages and gates source processing for tool results by tool name. A nil +// policy retains the legacy generic behavior for package-internal callers. func (r *sourceRegistry) EncodeMessagesWithPolicies( messages []chat.Message, argumentPolicy toolArgumentPolicy, @@ -523,27 +513,32 @@ func (r *sourceRegistry) registerToolArgumentValue(key string, value interface{} } // registerSourceIDByKey is the single key→source-space dispatch used for tool -// arguments, structured tool results, and database rows. Keeping one dispatch -// stops the recognized key set (and the http/https guard for web references) -// from drifting between call sites; it must stay in sync with decodableKeys. +// arguments, structured tool results, and database rows. It is driven by +// sourceKeySpaces — the same table that gates alias decode — so the recognized +// key set (and the http/https guard for web references) cannot drift between +// registration and decoding. func (r *sourceRegistry) registerSourceIDByKey(key, value string) { value = strings.TrimSpace(value) if value == "" || shortSourceAliasRE.MatchString(value) { return } - switch strings.ToLower(key) { - case "chunk_id", "faq_id", "chunk_ids", "faq_ids": + space, ok := sourceKeySpaces[strings.ToLower(key)] + if !ok { + return + } + switch space { + case spaceChunk: r.RegisterChunk(ChunkReference{ChunkID: value}) - case "knowledge_id", "knowledge_ids", "suspected_knowledge_ids": + case spaceDocument: r.RegisterDocument(value) - case "source_refs": + case spaceDocumentRef: // Stored refs use "knowledgeID|title"; only the ID part is durable. r.RegisterDocument(strings.TrimSpace(strings.SplitN(value, "|", 2)[0])) - case "knowledge_base", "knowledge_base_id", "knowledge_base_ids", "kb_id", "kb_ids": + case spaceKnowledgeBase: r.RegisterKnowledgeBase(value) - case "url", "urls": + case spaceWeb: // Only public web pages become web references. Internal schemes - // (res://, storage providers) must never enter the web alias space, + // (res://, storage providers) must never enter the web handle space, // where CompactKnownText would rewrite them a second time. if parsed, err := url.Parse(value); err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") { r.RegisterWeb(value, "") @@ -567,18 +562,6 @@ func (r *sourceRegistry) decodeJSONWithPolicy(raw string, encode bool, allowed f return string(encoded) } -// decodableKeys mirrors the ID-bearing keys recognized by -// registerToolArgumentValue. Alias -> real substitution on decode is restricted -// to these keys so that free-text values (e.g. a grep/search query that happens -// to equal "d1") are never rewritten into internal identifiers. -var decodableKeys = map[string]struct{}{ - - "chunk_id": {}, "faq_id": {}, "chunk_ids": {}, "faq_ids": {}, - "knowledge_id": {}, "knowledge_ids": {}, "suspected_knowledge_ids": {}, "source_refs": {}, - "knowledge_base": {}, "knowledge_base_id": {}, "knowledge_base_ids": {}, "kb_id": {}, "kb_ids": {}, - "url": {}, "urls": {}, -} - func (r *sourceRegistry) walkJSON(key string, value interface{}, encode bool, allowed func(string) bool) interface{} { switch typed := value.(type) { case string: @@ -595,7 +578,7 @@ func (r *sourceRegistry) walkJSON(key string, value interface{}, encode bool, al } // Decode only ID-bearing keys, and only when the value is alias-shaped, // so ordinary strings that coincidentally equal an alias are preserved. - if _, ok := decodableKeys[strings.ToLower(key)]; !ok { + if _, ok := sourceKeySpaces[strings.ToLower(key)]; !ok { return typed } if !shortSourceAliasRE.MatchString(strings.TrimSpace(typed)) { diff --git a/internal/modelcontext/sources_test.go b/internal/modelcontext/sources_test.go index ea052e88..3eca204c 100644 --- a/internal/modelcontext/sources_test.go +++ b/internal/modelcontext/sources_test.go @@ -67,41 +67,42 @@ func TestRegistrySuppressesSourceCitationsWhenDisabled(t *testing.T) { } func TestRegistryDecodesAliasesInNestedToolArguments(t *testing.T) { - registry := newSourceRegistry() + registry := NewRegistry(true) registry.RegisterDocument("knowledge-uuid-1") - registry.RegisterKnowledgeBase("kb-uuid-1") - registry.RegisterWeb("https://example.com/page", "Example") registry.RegisterChunk(ChunkReference{ChunkID: "chunk-uuid-1"}) + // Handles nested inside arrays and objects must decode as long as the key + // belongs to the named tool's declared source contract. calls := []types.LLMToolCall{{ Function: types.FunctionCall{ - Arguments: `{"knowledge_id":"d1","knowledge_base_ids":["b1"],"url":"w1","chunk_id":"c1"}`, + Name: "list_knowledge_chunks", + Arguments: `{"knowledge_id":"d1","filters":{"chunk_id":"c1","faq_id":["c1"]}}`, }, }} registry.DecodeToolCalls(calls) require.JSONEq(t, - `{"knowledge_id":"knowledge-uuid-1","knowledge_base_ids":["kb-uuid-1"],"url":"https://example.com/page","chunk_id":"chunk-uuid-1"}`, + `{"knowledge_id":"knowledge-uuid-1","filters":{"chunk_id":"chunk-uuid-1","faq_id":["chunk-uuid-1"]}}`, calls[0].Function.Arguments, ) } func TestDecodeToolCallsOnlyRewritesAliasBearingKeys(t *testing.T) { - registry := newSourceRegistry() - registry.RegisterDocument("knowledge-uuid-1") - registry.RegisterChunk(ChunkReference{ChunkID: "chunk-uuid-1"}) + registry := NewRegistry(true) + registry.RegisterKnowledgeBase("kb-uuid-1") - // A free-text field (query) whose value coincidentally equals an alias must - // be preserved verbatim, while ID-bearing keys still resolve to real IDs. + // A free-text field (query) whose value coincidentally equals a handle must + // be preserved verbatim, while declared ID-bearing keys resolve to real IDs. calls := []types.LLMToolCall{{ Function: types.FunctionCall{ - Arguments: `{"query":"d1","knowledge_id":"d1","content":"see c1 for details","chunk_id":"c1"}`, + Name: "knowledge_search", + Arguments: `{"query":"b1","content":"see b1 for details","knowledge_base_ids":["b1"]}`, }, }} registry.DecodeToolCalls(calls) require.JSONEq(t, - `{"query":"d1","knowledge_id":"knowledge-uuid-1","content":"see c1 for details","chunk_id":"chunk-uuid-1"}`, + `{"query":"b1","content":"see b1 for details","knowledge_base_ids":["kb-uuid-1"]}`, calls[0].Function.Arguments, ) } @@ -131,7 +132,7 @@ func TestEncodeMessagesCompactsCanonicalCitationsFromHistory(t *testing.T) { `web `, }} - encoded := registry.EncodeMessages(messages) + encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil) require.Equal(t, `Knowledge ; web `, encoded[0].Content) require.NotContains(t, encoded[0].Content, "chunk-real") require.NotContains(t, encoded[0].Content, "https://example.com") @@ -165,7 +166,7 @@ func TestEncodeMessagesMigratesLegacyToolHistoryAtReadTime(t *testing.T) { }, } - encoded := registry.EncodeMessages(messages) + encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil) require.JSONEq(t, `{"knowledge_base_ids":["b1"],"knowledge_ids":["d1"]}`, encoded[0].ToolCalls[0].Function.Arguments) require.Contains(t, encoded[1].Content, `chunk_id="c1"`) @@ -185,7 +186,7 @@ func TestEncodeMessagesDoesNotTreatLegacyPromptExampleAsARealSource(t *testing.T Content: `Old rule: cite `, }} - encoded := registry.EncodeMessages(messages) + encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil) require.Equal(t, messages[0].Content, encoded[0].Content) require.Zero(t, registry.Count()) } @@ -303,9 +304,13 @@ func TestModelOutputCompactsLabeledWikiReferences(t *testing.T) { require.Contains(t, output, `b1`) require.Contains(t, output, `knowledge_id="d1"`) - toolCalls := []types.LLMToolCall{{Function: types.FunctionCall{Arguments: `{"knowledge_id":"d1","knowledge_base_id":"b1"}`}}} - registry.DecodeToolCalls(toolCalls) - require.JSONEq(t, `{"knowledge_id":"doc-real-id","knowledge_base_id":"kb-real-id"}`, toolCalls[0].Function.Arguments) + toolCalls := []types.LLMToolCall{ + {Function: types.FunctionCall{Name: "wiki_read_source_doc", Arguments: `{"knowledge_id":"d1"}`}}, + {Function: types.FunctionCall{Name: "wiki_search", Arguments: `{"knowledge_base_id":"b1"}`}}, + } + registry.DecodeToolCallsWithPolicy(toolCalls, sourceArgumentAllowed) + require.JSONEq(t, `{"knowledge_id":"doc-real-id"}`, toolCalls[0].Function.Arguments) + require.JSONEq(t, `{"knowledge_base_id":"kb-real-id"}`, toolCalls[1].Function.Arguments) } func TestModelOutputGraphResultsUseChunkAliases(t *testing.T) { diff --git a/internal/modelcontext/tool_policy.go b/internal/modelcontext/tool_policy.go index fac2f40b..33b1b113 100644 --- a/internal/modelcontext/tool_policy.go +++ b/internal/modelcontext/tool_policy.go @@ -18,6 +18,32 @@ const ( var issueHandleShapeRE = regexp.MustCompile(`^i[1-9][0-9]*$`) +// sourceKeySpace identifies which source-handle space an ID-bearing JSON key +// belongs to. +type sourceKeySpace int + +const ( + spaceChunk sourceKeySpace = iota + spaceDocument + spaceDocumentRef // "knowledgeID|title" stored refs; only the ID is durable + spaceKnowledgeBase + spaceWeb +) + +// sourceKeySpaces is the single table of ID-bearing keys the source codec +// understands. It drives both handle registration (registerSourceIDByKey) and +// the alias-shaped-value decode gate (walkJSON), so registration and decoding +// cannot drift apart. Every key referenced by a toolHandlePolicies sourceIDKeys +// set must appear here (enforced by a test). +var sourceKeySpaces = map[string]sourceKeySpace{ + "chunk_id": spaceChunk, "faq_id": spaceChunk, "chunk_ids": spaceChunk, "faq_ids": spaceChunk, + "knowledge_id": spaceDocument, "knowledge_ids": spaceDocument, "suspected_knowledge_ids": spaceDocument, + "source_refs": spaceDocumentRef, + "knowledge_base": spaceKnowledgeBase, "knowledge_base_id": spaceKnowledgeBase, + "knowledge_base_ids": spaceKnowledgeBase, "kb_id": spaceKnowledgeBase, "kb_ids": spaceKnowledgeBase, + "url": spaceWeb, "urls": spaceWeb, +} + type toolHandlePolicy struct { sourceIDKeys map[string]struct{} sourceTextKeys map[string]struct{} diff --git a/internal/modelcontext/tool_policy_test.go b/internal/modelcontext/tool_policy_test.go new file mode 100644 index 00000000..8213021b --- /dev/null +++ b/internal/modelcontext/tool_policy_test.go @@ -0,0 +1,21 @@ +package modelcontext + +import "testing" + +// Every key a tool declares as a source-ID argument must have a registration +// space, otherwise its values could be decoded from handles but never +// registered (or vice versa) — the drift this table exists to prevent. +func TestEverySourceIDKeyHasARegistrationSpace(t *testing.T) { + for tool, policy := range toolHandlePolicies { + for key := range policy.sourceIDKeys { + if _, ok := sourceKeySpaces[key]; !ok { + t.Errorf("tool %s declares source key %q with no entry in sourceKeySpaces", tool, key) + } + } + for key := range policy.sourceTextKeys { + if _, ok := sourceKeySpaces[key]; ok { + t.Errorf("tool %s declares %q as free text but sourceKeySpaces also claims it as an ID key", tool, key) + } + } + } +}