refactor(modelcontext): unify source-key gating behind one dispatch table
sourceKeySpaces becomes the single table of ID-bearing keys: it drives handle registration (registerSourceIDByKey) and the alias-decode gate (walkJSON / unresolved-handle audit), replacing the separate decodableKeys list that had to be kept in sync by hand. The per-tool sourceIDKeys policy remains the outer gate; a new test asserts every policy key has a registration space. The generic nil-policy wrappers (sourceRegistry.DecodeToolCalls, EncodeMessages) are removed; the two tests that exercised anonymous-tool decoding now go through the public Registry with real tool names.main
parent
3b9de5de8e
commit
286eadac0a
|
|
@ -258,16 +258,10 @@ func (r *sourceRegistry) ChunkAlias(id string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
// DecodeToolCalls restores exact alias-valued JSON strings before tools parse
|
||||
// or validate their arguments. It never performs substring replacement.
|
||||
func (r *sourceRegistry) DecodeToolCalls(toolCalls []types.LLMToolCall) {
|
||||
r.DecodeToolCallsWithPolicy(toolCalls, nil)
|
||||
}
|
||||
|
||||
// toolArgumentPolicy decides whether a source-bearing JSON key belongs to a
|
||||
// particular tool contract. A nil policy retains the codec's legacy generic
|
||||
// behavior; application request lifecycles should provide an explicit policy
|
||||
// through Registry.
|
||||
// particular tool contract. Request lifecycles always pass the per-tool policy
|
||||
// (sourceArgumentAllowed); a nil policy allows every key and exists only for
|
||||
// package-internal replay paths that predate per-tool contracts.
|
||||
type toolArgumentPolicy func(toolName, key string) bool
|
||||
|
||||
// DecodeToolCallsWithPolicy restores aliases only for fields explicitly owned
|
||||
|
|
@ -319,7 +313,7 @@ func (r *sourceRegistry) collectUnresolvedToolHandles(
|
|||
switch typed := value.(type) {
|
||||
case string:
|
||||
key = strings.ToLower(key)
|
||||
if _, ok := decodableKeys[key]; !ok || !allowed(key) {
|
||||
if _, ok := sourceKeySpaces[key]; !ok || !allowed(key) {
|
||||
return
|
||||
}
|
||||
handle := strings.TrimSpace(typed)
|
||||
|
|
@ -337,13 +331,9 @@ func (r *sourceRegistry) collectUnresolvedToolHandles(
|
|||
}
|
||||
}
|
||||
|
||||
// EncodeMessages compacts known real identifiers in assistant tool-call replay.
|
||||
func (r *sourceRegistry) EncodeMessages(messages []chat.Message) []chat.Message {
|
||||
return r.EncodeMessagesWithPolicies(messages, nil, nil)
|
||||
}
|
||||
|
||||
// EncodeMessagesWithPolicies additionally gates source processing for tool
|
||||
// result messages. A nil result policy retains the legacy generic behavior.
|
||||
// EncodeMessagesWithPolicies compacts known real identifiers in replayed
|
||||
// messages and gates source processing for tool results by tool name. A nil
|
||||
// policy retains the legacy generic behavior for package-internal callers.
|
||||
func (r *sourceRegistry) EncodeMessagesWithPolicies(
|
||||
messages []chat.Message,
|
||||
argumentPolicy toolArgumentPolicy,
|
||||
|
|
@ -523,27 +513,32 @@ func (r *sourceRegistry) registerToolArgumentValue(key string, value interface{}
|
|||
}
|
||||
|
||||
// registerSourceIDByKey is the single key→source-space dispatch used for tool
|
||||
// arguments, structured tool results, and database rows. Keeping one dispatch
|
||||
// stops the recognized key set (and the http/https guard for web references)
|
||||
// from drifting between call sites; it must stay in sync with decodableKeys.
|
||||
// arguments, structured tool results, and database rows. It is driven by
|
||||
// sourceKeySpaces — the same table that gates alias decode — so the recognized
|
||||
// key set (and the http/https guard for web references) cannot drift between
|
||||
// registration and decoding.
|
||||
func (r *sourceRegistry) registerSourceIDByKey(key, value string) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || shortSourceAliasRE.MatchString(value) {
|
||||
return
|
||||
}
|
||||
switch strings.ToLower(key) {
|
||||
case "chunk_id", "faq_id", "chunk_ids", "faq_ids":
|
||||
space, ok := sourceKeySpaces[strings.ToLower(key)]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch space {
|
||||
case spaceChunk:
|
||||
r.RegisterChunk(ChunkReference{ChunkID: value})
|
||||
case "knowledge_id", "knowledge_ids", "suspected_knowledge_ids":
|
||||
case spaceDocument:
|
||||
r.RegisterDocument(value)
|
||||
case "source_refs":
|
||||
case spaceDocumentRef:
|
||||
// Stored refs use "knowledgeID|title"; only the ID part is durable.
|
||||
r.RegisterDocument(strings.TrimSpace(strings.SplitN(value, "|", 2)[0]))
|
||||
case "knowledge_base", "knowledge_base_id", "knowledge_base_ids", "kb_id", "kb_ids":
|
||||
case spaceKnowledgeBase:
|
||||
r.RegisterKnowledgeBase(value)
|
||||
case "url", "urls":
|
||||
case spaceWeb:
|
||||
// Only public web pages become web references. Internal schemes
|
||||
// (res://, storage providers) must never enter the web alias space,
|
||||
// (res://, storage providers) must never enter the web handle space,
|
||||
// where CompactKnownText would rewrite them a second time.
|
||||
if parsed, err := url.Parse(value); err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") {
|
||||
r.RegisterWeb(value, "")
|
||||
|
|
@ -567,18 +562,6 @@ func (r *sourceRegistry) decodeJSONWithPolicy(raw string, encode bool, allowed f
|
|||
return string(encoded)
|
||||
}
|
||||
|
||||
// decodableKeys mirrors the ID-bearing keys recognized by
|
||||
// registerToolArgumentValue. Alias -> real substitution on decode is restricted
|
||||
// to these keys so that free-text values (e.g. a grep/search query that happens
|
||||
// to equal "d1") are never rewritten into internal identifiers.
|
||||
var decodableKeys = map[string]struct{}{
|
||||
|
||||
"chunk_id": {}, "faq_id": {}, "chunk_ids": {}, "faq_ids": {},
|
||||
"knowledge_id": {}, "knowledge_ids": {}, "suspected_knowledge_ids": {}, "source_refs": {},
|
||||
"knowledge_base": {}, "knowledge_base_id": {}, "knowledge_base_ids": {}, "kb_id": {}, "kb_ids": {},
|
||||
"url": {}, "urls": {},
|
||||
}
|
||||
|
||||
func (r *sourceRegistry) walkJSON(key string, value interface{}, encode bool, allowed func(string) bool) interface{} {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
|
|
@ -595,7 +578,7 @@ func (r *sourceRegistry) walkJSON(key string, value interface{}, encode bool, al
|
|||
}
|
||||
// Decode only ID-bearing keys, and only when the value is alias-shaped,
|
||||
// so ordinary strings that coincidentally equal an alias are preserved.
|
||||
if _, ok := decodableKeys[strings.ToLower(key)]; !ok {
|
||||
if _, ok := sourceKeySpaces[strings.ToLower(key)]; !ok {
|
||||
return typed
|
||||
}
|
||||
if !shortSourceAliasRE.MatchString(strings.TrimSpace(typed)) {
|
||||
|
|
|
|||
|
|
@ -67,41 +67,42 @@ func TestRegistrySuppressesSourceCitationsWhenDisabled(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRegistryDecodesAliasesInNestedToolArguments(t *testing.T) {
|
||||
registry := newSourceRegistry()
|
||||
registry := NewRegistry(true)
|
||||
registry.RegisterDocument("knowledge-uuid-1")
|
||||
registry.RegisterKnowledgeBase("kb-uuid-1")
|
||||
registry.RegisterWeb("https://example.com/page", "Example")
|
||||
registry.RegisterChunk(ChunkReference{ChunkID: "chunk-uuid-1"})
|
||||
|
||||
// Handles nested inside arrays and objects must decode as long as the key
|
||||
// belongs to the named tool's declared source contract.
|
||||
calls := []types.LLMToolCall{{
|
||||
Function: types.FunctionCall{
|
||||
Arguments: `{"knowledge_id":"d1","knowledge_base_ids":["b1"],"url":"w1","chunk_id":"c1"}`,
|
||||
Name: "list_knowledge_chunks",
|
||||
Arguments: `{"knowledge_id":"d1","filters":{"chunk_id":"c1","faq_id":["c1"]}}`,
|
||||
},
|
||||
}}
|
||||
registry.DecodeToolCalls(calls)
|
||||
|
||||
require.JSONEq(t,
|
||||
`{"knowledge_id":"knowledge-uuid-1","knowledge_base_ids":["kb-uuid-1"],"url":"https://example.com/page","chunk_id":"chunk-uuid-1"}`,
|
||||
`{"knowledge_id":"knowledge-uuid-1","filters":{"chunk_id":"chunk-uuid-1","faq_id":["chunk-uuid-1"]}}`,
|
||||
calls[0].Function.Arguments,
|
||||
)
|
||||
}
|
||||
|
||||
func TestDecodeToolCallsOnlyRewritesAliasBearingKeys(t *testing.T) {
|
||||
registry := newSourceRegistry()
|
||||
registry.RegisterDocument("knowledge-uuid-1")
|
||||
registry.RegisterChunk(ChunkReference{ChunkID: "chunk-uuid-1"})
|
||||
registry := NewRegistry(true)
|
||||
registry.RegisterKnowledgeBase("kb-uuid-1")
|
||||
|
||||
// A free-text field (query) whose value coincidentally equals an alias must
|
||||
// be preserved verbatim, while ID-bearing keys still resolve to real IDs.
|
||||
// A free-text field (query) whose value coincidentally equals a handle must
|
||||
// be preserved verbatim, while declared ID-bearing keys resolve to real IDs.
|
||||
calls := []types.LLMToolCall{{
|
||||
Function: types.FunctionCall{
|
||||
Arguments: `{"query":"d1","knowledge_id":"d1","content":"see c1 for details","chunk_id":"c1"}`,
|
||||
Name: "knowledge_search",
|
||||
Arguments: `{"query":"b1","content":"see b1 for details","knowledge_base_ids":["b1"]}`,
|
||||
},
|
||||
}}
|
||||
registry.DecodeToolCalls(calls)
|
||||
|
||||
require.JSONEq(t,
|
||||
`{"query":"d1","knowledge_id":"knowledge-uuid-1","content":"see c1 for details","chunk_id":"chunk-uuid-1"}`,
|
||||
`{"query":"b1","content":"see b1 for details","knowledge_base_ids":["kb-uuid-1"]}`,
|
||||
calls[0].Function.Arguments,
|
||||
)
|
||||
}
|
||||
|
|
@ -131,7 +132,7 @@ func TestEncodeMessagesCompactsCanonicalCitationsFromHistory(t *testing.T) {
|
|||
`web <web url="https://example.com/a?x=1&y=2" title="Example & More" />`,
|
||||
}}
|
||||
|
||||
encoded := registry.EncodeMessages(messages)
|
||||
encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil)
|
||||
require.Equal(t, `Knowledge <ref id="c1"/>; web <ref id="w1"/>`, encoded[0].Content)
|
||||
require.NotContains(t, encoded[0].Content, "chunk-real")
|
||||
require.NotContains(t, encoded[0].Content, "https://example.com")
|
||||
|
|
@ -165,7 +166,7 @@ func TestEncodeMessagesMigratesLegacyToolHistoryAtReadTime(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
encoded := registry.EncodeMessages(messages)
|
||||
encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil)
|
||||
require.JSONEq(t, `{"knowledge_base_ids":["b1"],"knowledge_ids":["d1"]}`,
|
||||
encoded[0].ToolCalls[0].Function.Arguments)
|
||||
require.Contains(t, encoded[1].Content, `chunk_id="c1"`)
|
||||
|
|
@ -185,7 +186,7 @@ func TestEncodeMessagesDoesNotTreatLegacyPromptExampleAsARealSource(t *testing.T
|
|||
Content: `Old rule: cite <kb doc="..." chunk_id="..." />`,
|
||||
}}
|
||||
|
||||
encoded := registry.EncodeMessages(messages)
|
||||
encoded := registry.EncodeMessagesWithPolicies(messages, nil, nil)
|
||||
require.Equal(t, messages[0].Content, encoded[0].Content)
|
||||
require.Zero(t, registry.Count())
|
||||
}
|
||||
|
|
@ -303,9 +304,13 @@ func TestModelOutputCompactsLabeledWikiReferences(t *testing.T) {
|
|||
require.Contains(t, output, `<knowledge_base_id>b1</knowledge_base_id>`)
|
||||
require.Contains(t, output, `knowledge_id="d1"`)
|
||||
|
||||
toolCalls := []types.LLMToolCall{{Function: types.FunctionCall{Arguments: `{"knowledge_id":"d1","knowledge_base_id":"b1"}`}}}
|
||||
registry.DecodeToolCalls(toolCalls)
|
||||
require.JSONEq(t, `{"knowledge_id":"doc-real-id","knowledge_base_id":"kb-real-id"}`, toolCalls[0].Function.Arguments)
|
||||
toolCalls := []types.LLMToolCall{
|
||||
{Function: types.FunctionCall{Name: "wiki_read_source_doc", Arguments: `{"knowledge_id":"d1"}`}},
|
||||
{Function: types.FunctionCall{Name: "wiki_search", Arguments: `{"knowledge_base_id":"b1"}`}},
|
||||
}
|
||||
registry.DecodeToolCallsWithPolicy(toolCalls, sourceArgumentAllowed)
|
||||
require.JSONEq(t, `{"knowledge_id":"doc-real-id"}`, toolCalls[0].Function.Arguments)
|
||||
require.JSONEq(t, `{"knowledge_base_id":"kb-real-id"}`, toolCalls[1].Function.Arguments)
|
||||
}
|
||||
|
||||
func TestModelOutputGraphResultsUseChunkAliases(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,32 @@ const (
|
|||
|
||||
var issueHandleShapeRE = regexp.MustCompile(`^i[1-9][0-9]*$`)
|
||||
|
||||
// sourceKeySpace identifies which source-handle space an ID-bearing JSON key
|
||||
// belongs to.
|
||||
type sourceKeySpace int
|
||||
|
||||
const (
|
||||
spaceChunk sourceKeySpace = iota
|
||||
spaceDocument
|
||||
spaceDocumentRef // "knowledgeID|title" stored refs; only the ID is durable
|
||||
spaceKnowledgeBase
|
||||
spaceWeb
|
||||
)
|
||||
|
||||
// sourceKeySpaces is the single table of ID-bearing keys the source codec
|
||||
// understands. It drives both handle registration (registerSourceIDByKey) and
|
||||
// the alias-shaped-value decode gate (walkJSON), so registration and decoding
|
||||
// cannot drift apart. Every key referenced by a toolHandlePolicies sourceIDKeys
|
||||
// set must appear here (enforced by a test).
|
||||
var sourceKeySpaces = map[string]sourceKeySpace{
|
||||
"chunk_id": spaceChunk, "faq_id": spaceChunk, "chunk_ids": spaceChunk, "faq_ids": spaceChunk,
|
||||
"knowledge_id": spaceDocument, "knowledge_ids": spaceDocument, "suspected_knowledge_ids": spaceDocument,
|
||||
"source_refs": spaceDocumentRef,
|
||||
"knowledge_base": spaceKnowledgeBase, "knowledge_base_id": spaceKnowledgeBase,
|
||||
"knowledge_base_ids": spaceKnowledgeBase, "kb_id": spaceKnowledgeBase, "kb_ids": spaceKnowledgeBase,
|
||||
"url": spaceWeb, "urls": spaceWeb,
|
||||
}
|
||||
|
||||
type toolHandlePolicy struct {
|
||||
sourceIDKeys map[string]struct{}
|
||||
sourceTextKeys map[string]struct{}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
package modelcontext
|
||||
|
||||
import "testing"
|
||||
|
||||
// Every key a tool declares as a source-ID argument must have a registration
|
||||
// space, otherwise its values could be decoded from handles but never
|
||||
// registered (or vice versa) — the drift this table exists to prevent.
|
||||
func TestEverySourceIDKeyHasARegistrationSpace(t *testing.T) {
|
||||
for tool, policy := range toolHandlePolicies {
|
||||
for key := range policy.sourceIDKeys {
|
||||
if _, ok := sourceKeySpaces[key]; !ok {
|
||||
t.Errorf("tool %s declares source key %q with no entry in sourceKeySpaces", tool, key)
|
||||
}
|
||||
}
|
||||
for key := range policy.sourceTextKeys {
|
||||
if _, ok := sourceKeySpaces[key]; ok {
|
||||
t.Errorf("tool %s declares %q as free text but sourceKeySpaces also claims it as an ID key", tool, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue