From 213ca0b42e8b3f0551c894dc8d37068a86e4e808 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 27 Jul 2026 01:13:08 +0800 Subject: [PATCH] refactor(modelcontext): rebuild every handle space on one generic table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleTable[M] is the single bidirectional primitive behind all model handles: source chunks/documents/KBs/web pages, res:// resource handles, and the exported HandleTable (iN issue handles, ingest ref-N/c000 handles). Metadata lives under the same lock as the mapping, web references dedup on the canonical URL while decoding to the raw URL, and the append-only counter preserves historical numbering exactly. Behavior is unchanged and proven by the moved test suites; the per-space text-rewrite strategies (plain substring for resources, word-boundary regex for HandleTable, shape-regex lookup for sources) deliberately stay in their wrappers because each is load-bearing wire behavior. The unused ChunkReference.Alias field is dropped — handles are derived from the table, never stored in metadata. --- internal/modelcontext/citations.go | 12 +- internal/modelcontext/handle_table.go | 137 ++++++++++++++++ internal/modelcontext/handles.go | 97 +++--------- internal/modelcontext/resources.go | 65 +++----- internal/modelcontext/sources.go | 216 ++++++++------------------ internal/modelcontext/sources_test.go | 8 +- 6 files changed, 255 insertions(+), 280 deletions(-) create mode 100644 internal/modelcontext/handle_table.go diff --git a/internal/modelcontext/citations.go b/internal/modelcontext/citations.go index 14a26430..d31d9b36 100644 --- a/internal/modelcontext/citations.go +++ b/internal/modelcontext/citations.go @@ -172,19 +172,15 @@ func (r *sourceRegistry) ExpandText(text string) string { return "" } alias := strings.ToLower(match[1]) - r.mu.RLock() - chunkRef := r.chunkByAlias[alias] - webRef := r.webByAlias[alias] - r.mu.RUnlock() - if chunkRef != nil { - attrs := fmt.Sprintf(`doc="%s" chunk_id="%s"`, escapeAttr(chunkRef.DocumentTitle), escapeAttr(chunkRef.ChunkID)) + if chunkID, chunkRef, ok := r.chunks.resolve(alias); ok { + attrs := fmt.Sprintf(`doc="%s" chunk_id="%s"`, escapeAttr(chunkRef.DocumentTitle), escapeAttr(chunkID)) if chunkRef.KnowledgeBaseID != "" { attrs += fmt.Sprintf(` kb_id="%s"`, escapeAttr(chunkRef.KnowledgeBaseID)) } return "" } - if webRef != nil { - return fmt.Sprintf(``, escapeAttr(webRef.URL), escapeAttr(webRef.Title)) + if rawURL, web, ok := r.webs.resolve(alias); ok { + return fmt.Sprintf(``, escapeAttr(rawURL), escapeAttr(web.title)) } return "" }) diff --git a/internal/modelcontext/handle_table.go b/internal/modelcontext/handle_table.go new file mode 100644 index 00000000..72c3d7f2 --- /dev/null +++ b/internal/modelcontext/handle_table.go @@ -0,0 +1,137 @@ +package modelcontext + +import ( + "fmt" + "sync" +) + +// handleTable is the single bidirectional primitive behind every model-handle +// space: cN/dN/bN/wN source handles, iN issue handles, res://NNNN resource +// handles, and the ref-N / c000 ingest handles exposed through HandleTable. +// +// It maps a durable dedup key to a sequentially allocated handle and stores +// the durable value plus optional metadata under one lock, so a resolvable +// handle can never be observed without its metadata. The key and the value +// may differ (web references dedup on a canonicalized URL but decode back to +// the original raw URL). Entries are never removed, which makes the counter +// equivalent to the historical len(map)+1 allocation and keeps handle +// numbering stable for the lifetime of the table. +type handleTable[M any] struct { + mu sync.RWMutex + prefix string + width int + next int + handleByKey map[string]string + entryByHandle map[string]*handleEntry[M] +} + +type handleEntry[M any] struct { + value string // durable value a handle decodes back to + meta M +} + +// handlePair is a snapshot row for text codecs (compaction/decoding). +type handlePair struct { + value string + handle string +} + +// newHandleTable creates a handle space such as c1 (prefix="c", width=0, +// start=1), c000 (prefix="c", width=3, start=0) or res://0001 +// (prefix="res://", width=4, start=1). +func newHandleTable[M any](prefix string, width, start int) *handleTable[M] { + return &handleTable[M]{ + prefix: prefix, + width: width, + next: start, + handleByKey: make(map[string]string), + entryByHandle: make(map[string]*handleEntry[M]), + } +} + +// register returns the handle assigned to key, allocating the next one on +// first use. value is what the handle decodes back to; merge, when non-nil, +// folds the metadata of a repeated registration into the existing entry. +func (t *handleTable[M]) register(key, value string, meta M, merge func(dst *M, src M)) string { + if t == nil || key == "" { + return "" + } + t.mu.Lock() + defer t.mu.Unlock() + if handle, ok := t.handleByKey[key]; ok { + if merge != nil { + merge(&t.entryByHandle[handle].meta, meta) + } + return handle + } + number := fmt.Sprintf("%d", t.next) + if t.width > 0 { + number = fmt.Sprintf("%0*d", t.width, t.next) + } + t.next++ + handle := t.prefix + number + t.handleByKey[key] = handle + t.entryByHandle[handle] = &handleEntry[M]{value: value, meta: meta} + return handle +} + +// handleForKey returns an existing handle without allocating one. +func (t *handleTable[M]) handleForKey(key string) (string, bool) { + if t == nil { + return "", false + } + t.mu.RLock() + defer t.mu.RUnlock() + handle, ok := t.handleByKey[key] + return handle, ok +} + +// resolve returns the durable value and metadata snapshot for a known handle. +func (t *handleTable[M]) resolve(handle string) (string, M, bool) { + var zero M + if t == nil { + return "", zero, false + } + t.mu.RLock() + defer t.mu.RUnlock() + entry, ok := t.entryByHandle[handle] + if !ok { + return "", zero, false + } + return entry.value, entry.meta, true +} + +// has reports whether handle exists in this table. +func (t *handleTable[M]) has(handle string) bool { + if t == nil { + return false + } + t.mu.RLock() + defer t.mu.RUnlock() + _, ok := t.entryByHandle[handle] + return ok +} + +func (t *handleTable[M]) size() int { + if t == nil { + return 0 + } + t.mu.RLock() + defer t.mu.RUnlock() + return len(t.entryByHandle) +} + +// pairs returns a value/handle snapshot for text codecs. Callers own ordering +// (e.g. longest-value-first for substring compaction). +func (t *handleTable[M]) pairs() []handlePair { + if t == nil { + return nil + } + t.mu.RLock() + defer t.mu.RUnlock() + out := make([]handlePair, 0, len(t.entryByHandle)) + for handle, entry := range t.entryByHandle { + out = append(out, handlePair{value: entry.value, handle: handle}) + } + return out +} diff --git a/internal/modelcontext/handles.go b/internal/modelcontext/handles.go index cf0dc6f5..958caad8 100644 --- a/internal/modelcontext/handles.go +++ b/internal/modelcontext/handles.go @@ -1,35 +1,24 @@ package modelcontext import ( - "fmt" "regexp" "sort" "strings" - "sync" ) // HandleTable is a typed, invocation-local bidirectional mapping used when a -// prompt needs compact handles for durable values. Handles must never be -// persisted; Resolve converts model output back to the durable value first. +// prompt needs compact handles for durable values (wiki issue iN handles, +// ingest ref-N slug handles, c000 citation-batch handles). Handles must never +// be persisted; Resolve converts model output back to the durable value first. +// It is the exported word-boundary-aware wrapper over handleTable. type HandleTable struct { - mu sync.RWMutex - prefix string - width int - next int - toHandle map[string]string - toValue map[string]string + table *handleTable[struct{}] } // NewHandleTable creates a handle space such as c000 (prefix=c, width=3, // start=0) or ref-1 (prefix=ref-, width=0, start=1). func NewHandleTable(prefix string, width, start int) *HandleTable { - return &HandleTable{ - prefix: prefix, - width: width, - next: start, - toHandle: make(map[string]string), - toValue: make(map[string]string), - } + return &HandleTable{table: newHandleTable[struct{}](prefix, width, start)} } // Register returns the stable handle assigned to value in this table. @@ -41,20 +30,7 @@ func (t *HandleTable) Register(value string) string { if value == "" { return "" } - t.mu.Lock() - defer t.mu.Unlock() - if handle := t.toHandle[value]; handle != "" { - return handle - } - number := fmt.Sprintf("%d", t.next) - if t.width > 0 { - number = fmt.Sprintf("%0*d", t.width, t.next) - } - t.next++ - handle := t.prefix + number - t.toHandle[value] = handle - t.toValue[handle] = value - return handle + return t.table.register(value, value, struct{}{}, nil) } // Handle returns an already registered handle without creating one. @@ -62,10 +38,7 @@ func (t *HandleTable) Handle(value string) (string, bool) { if t == nil { return "", false } - t.mu.RLock() - defer t.mu.RUnlock() - handle, ok := t.toHandle[value] - return handle, ok + return t.table.handleForKey(value) } // Resolve converts a known model handle back to its durable value. @@ -73,28 +46,19 @@ func (t *HandleTable) Resolve(handle string) (string, bool) { if t == nil { return "", false } - t.mu.RLock() - defer t.mu.RUnlock() - value, ok := t.toValue[strings.TrimSpace(handle)] + value, _, ok := t.table.resolve(strings.TrimSpace(handle)) return value, ok } func (t *HandleTable) Empty() bool { - if t == nil { - return true - } - t.mu.RLock() - defer t.mu.RUnlock() - return len(t.toHandle) == 0 + return t == nil || t.table.size() == 0 } func (t *HandleTable) Len() int { if t == nil { return 0 } - t.mu.RLock() - defer t.mu.RUnlock() - return len(t.toHandle) + return t.table.size() } // EncodeKnownText replaces already-registered durable values with their model @@ -103,13 +67,7 @@ func (t *HandleTable) EncodeKnownText(value string) string { if t == nil || value == "" { return value } - t.mu.RLock() - type pair struct{ value, handle string } - pairs := make([]pair, 0, len(t.toHandle)) - for durable, handle := range t.toHandle { - pairs = append(pairs, pair{durable, handle}) - } - t.mu.RUnlock() + pairs := t.table.pairs() sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].value) > len(pairs[j].value) }) for _, item := range pairs { value = strings.ReplaceAll(value, item.value, item.handle) @@ -117,25 +75,21 @@ func (t *HandleTable) EncodeKnownText(value string) string { return value } -// DecodeKnownText restores registered handles in complete text. +// DecodeKnownText restores registered handles in complete text. Unlike the +// resource codec, replacement is word-boundary-aware so short handles such as +// i1 cannot fire inside ordinary prose tokens. func (t *HandleTable) DecodeKnownText(value string) string { if t == nil || value == "" { return value } - t.mu.RLock() - handles := make([]string, 0, len(t.toValue)) - for handle := range t.toValue { - handles = append(handles, handle) - } - t.mu.RUnlock() - sort.SliceStable(handles, func(i, j int) bool { return len(handles[i]) > len(handles[j]) }) - for _, handle := range handles { - pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(handle) + `\b`) - if durable, ok := t.Resolve(handle); ok { - // A durable value may legally contain '$'. ReplaceAllString would - // interpret it as a regexp expansion, so use a literal callback. - value = pattern.ReplaceAllStringFunc(value, func(string) string { return durable }) - } + pairs := t.table.pairs() + sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].handle) > len(pairs[j].handle) }) + for _, item := range pairs { + pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(item.handle) + `\b`) + // A durable value may legally contain '$'. ReplaceAllString would + // interpret it as a regexp expansion, so use a literal callback. + durable := item.value + value = pattern.ReplaceAllStringFunc(value, func(string) string { return durable }) } return value } @@ -165,9 +119,8 @@ func (d *HandleStreamDecoder) Feed(chunk string) string { start-- } tail := combined[start:] - d.table.mu.RLock() - prefix := d.table.prefix - d.table.mu.RUnlock() + // The prefix is immutable after construction, so no lock is needed. + prefix := d.table.table.prefix if couldBeNumericHandle(tail, prefix) { d.pending = tail combined = combined[:start] diff --git a/internal/modelcontext/resources.go b/internal/modelcontext/resources.go index 7046d75b..b92402be 100644 --- a/internal/modelcontext/resources.go +++ b/internal/modelcontext/resources.go @@ -4,11 +4,9 @@ package modelcontext import ( - "fmt" "regexp" "sort" "strings" - "sync" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" @@ -44,17 +42,15 @@ var aliasShapeRE = regexp.MustCompile(`res://\d+`) // resourceRegistry assigns low-entropy, request-local aliases to stable resource // handles. It is safe to reuse across all rounds of one Agent execution. type resourceRegistry struct { - mu sync.RWMutex - refToAlias map[string]string - aliasToRef map[string]string + table *handleTable[struct{}] } // newResourceRegistry creates an empty request-local alias registry. func newResourceRegistry() *resourceRegistry { - return &resourceRegistry{ - refToAlias: make(map[string]string), - aliasToRef: make(map[string]string), - } + // A URL-shaped handle (scheme://digits) keeps the token low-entropy while + // looking enough like a link that the model reuses it verbatim inside + // Markdown image/link syntax instead of reasoning about or rewriting it. + return &resourceRegistry{table: newHandleTable[struct{}]("res://", 4, 1)} } // EncodeText replaces stored references with compact, stable aliases. @@ -63,38 +59,23 @@ func (r *resourceRegistry) EncodeText(value string) string { return value } return storedRefRE.ReplaceAllStringFunc(value, func(ref string) string { - r.mu.Lock() - defer r.mu.Unlock() - if alias, ok := r.refToAlias[ref]; ok { - return alias - } - // A URL-shaped alias (scheme://digits) keeps the token low-entropy while - // looking enough like a link that the model reuses it verbatim inside - // Markdown image/link syntax instead of reasoning about or rewriting it. - alias := fmt.Sprintf("res://%04d", len(r.aliasToRef)+1) - r.refToAlias[ref] = alias - r.aliasToRef[alias] = ref - return alias + return r.table.register(ref, ref, struct{}{}, nil) }) } -// DecodeText restores every alias currently known to the registry. +// DecodeText restores every alias currently known to the registry. Aliases are +// replaced longest-first with no word-boundary check — this plain substring +// behavior is load-bearing for handles adjacent to punctuation in Markdown. func (r *resourceRegistry) DecodeText(value string) string { if r == nil || value == "" { return value } - r.mu.RLock() - aliases := make([]string, 0, len(r.aliasToRef)) - for alias := range r.aliasToRef { - aliases = append(aliases, alias) + pairs := r.table.pairs() + sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].handle) > len(pairs[j].handle) }) + for _, item := range pairs { + value = strings.ReplaceAll(value, item.handle, item.value) } - sort.SliceStable(aliases, func(i, j int) bool { return len(aliases[i]) > len(aliases[j]) }) - decoded := value - for _, alias := range aliases { - decoded = strings.ReplaceAll(decoded, alias, r.aliasToRef[alias]) - } - r.mu.RUnlock() - return decoded + return value } // StripOrphanAliases removes alias-shaped tokens after all known aliases have @@ -153,13 +134,8 @@ func (r *resourceRegistry) OrphanAliases(decoded string) []string { var orphans []string seen := make(map[string]struct{}) for _, match := range aliasShapeRE.FindAllString(decoded, -1) { - if r != nil { - r.mu.RLock() - _, known := r.aliasToRef[match] - r.mu.RUnlock() - if known { - continue - } + if r != nil && r.table.has(match) { + continue } if _, dup := seen[match]; dup { continue @@ -171,11 +147,10 @@ func (r *resourceRegistry) OrphanAliases(decoded string) []string { } func (r *resourceRegistry) aliases() []string { - r.mu.RLock() - defer r.mu.RUnlock() - aliases := make([]string, 0, len(r.aliasToRef)) - for alias := range r.aliasToRef { - aliases = append(aliases, alias) + pairs := r.table.pairs() + aliases := make([]string, 0, len(pairs)) + for _, item := range pairs { + aliases = append(aliases, item.handle) } return aliases } diff --git a/internal/modelcontext/sources.go b/internal/modelcontext/sources.go index 2f0369f5..cd711e68 100644 --- a/internal/modelcontext/sources.go +++ b/internal/modelcontext/sources.go @@ -7,19 +7,16 @@ package modelcontext import ( "encoding/json" - "fmt" "net/url" "regexp" "sort" "strings" - "sync" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" ) type ChunkReference struct { - Alias string ChunkID string KnowledgeID string KnowledgeBaseID string @@ -28,27 +25,20 @@ type ChunkReference struct { ChunkType string } -type WebReference struct { - Alias string - URL string - Title string +// webMeta is the per-web-page metadata stored next to the raw URL. +type webMeta struct { + title string } // sourceRegistry is scoped to one assistant response (including every Agent tool -// round). Aliases are never persisted or accepted across requests. +// round). Handles are never persisted or accepted across requests. type sourceRegistry struct { - mu sync.RWMutex - citationsEnabled bool - chunkByID map[string]*ChunkReference - chunkByAlias map[string]*ChunkReference - docToAlias map[string]string - aliasToDoc map[string]string - kbToAlias map[string]string - aliasToKB map[string]string - webByURL map[string]*WebReference - webByAlias map[string]*WebReference + chunks *handleTable[ChunkReference] + docs *handleTable[struct{}] + kbs *handleTable[struct{}] + webs *handleTable[webMeta] } func newSourceRegistry(citationsEnabled ...bool) *sourceRegistry { @@ -58,14 +48,10 @@ func newSourceRegistry(citationsEnabled ...bool) *sourceRegistry { } return &sourceRegistry{ citationsEnabled: enabled, - chunkByID: make(map[string]*ChunkReference), - chunkByAlias: make(map[string]*ChunkReference), - docToAlias: make(map[string]string), - aliasToDoc: make(map[string]string), - kbToAlias: make(map[string]string), - aliasToKB: make(map[string]string), - webByURL: make(map[string]*WebReference), - webByAlias: make(map[string]*WebReference), + chunks: newHandleTable[ChunkReference]("c", 0, 1), + docs: newHandleTable[struct{}]("d", 0, 1), + kbs: newHandleTable[struct{}]("b", 0, 1), + webs: newHandleTable[webMeta]("w", 0, 1), } } @@ -73,9 +59,18 @@ func (r *sourceRegistry) Count() int { if r == nil { return 0 } - r.mu.RLock() - defer r.mu.RUnlock() - return len(r.chunkByAlias) + len(r.webByAlias) + return r.chunks.size() + r.webs.size() +} + +// knownHandle implements the shared guard for handle-shaped registration +// input: a model-emitted handle is echoed back only when it already exists, +// and is never accepted as a new durable identity. +func knownHandle[M any](table *handleTable[M], id string) string { + handle := strings.ToLower(id) + if table.has(handle) { + return handle + } + return "" } func (r *sourceRegistry) RegisterChunk(ref ChunkReference) string { @@ -87,26 +82,9 @@ func (r *sourceRegistry) RegisterChunk(ref ChunkReference) string { return "" } if shortSourceAliasRE.MatchString(ref.ChunkID) { - alias := strings.ToLower(ref.ChunkID) - r.mu.RLock() - _, known := r.chunkByAlias[alias] - r.mu.RUnlock() - if known { - return alias - } - return "" + return knownHandle(r.chunks, ref.ChunkID) } - r.mu.Lock() - defer r.mu.Unlock() - if existing := r.chunkByID[ref.ChunkID]; existing != nil { - mergeChunkReference(existing, ref) - return existing.Alias - } - ref.Alias = fmt.Sprintf("c%d", len(r.chunkByAlias)+1) - copyRef := ref - r.chunkByID[ref.ChunkID] = ©Ref - r.chunkByAlias[ref.Alias] = ©Ref - return ref.Alias + return r.chunks.register(ref.ChunkID, ref.ChunkID, ref, mergeChunkReference) } func mergeChunkReference(dst *ChunkReference, src ChunkReference) { @@ -133,24 +111,9 @@ func (r *sourceRegistry) RegisterDocument(id string) string { return "" } if shortSourceAliasRE.MatchString(id) { - alias := strings.ToLower(id) - r.mu.RLock() - _, known := r.aliasToDoc[alias] - r.mu.RUnlock() - if known { - return alias - } - return "" + return knownHandle(r.docs, id) } - r.mu.Lock() - defer r.mu.Unlock() - if alias := r.docToAlias[id]; alias != "" { - return alias - } - alias := fmt.Sprintf("d%d", len(r.aliasToDoc)+1) - r.docToAlias[id] = alias - r.aliasToDoc[alias] = id - return alias + return r.docs.register(id, id, struct{}{}, nil) } func (r *sourceRegistry) RegisterKnowledgeBase(id string) string { @@ -159,24 +122,9 @@ func (r *sourceRegistry) RegisterKnowledgeBase(id string) string { return "" } if shortSourceAliasRE.MatchString(id) { - alias := strings.ToLower(id) - r.mu.RLock() - _, known := r.aliasToKB[alias] - r.mu.RUnlock() - if known { - return alias - } - return "" + return knownHandle(r.kbs, id) } - r.mu.Lock() - defer r.mu.Unlock() - if alias := r.kbToAlias[id]; alias != "" { - return alias - } - alias := fmt.Sprintf("b%d", len(r.aliasToKB)+1) - r.kbToAlias[id] = alias - r.aliasToKB[alias] = id - return alias + return r.kbs.register(id, id, struct{}{}, nil) } func (r *sourceRegistry) RegisterWeb(rawURL, title string) string { @@ -185,32 +133,15 @@ func (r *sourceRegistry) RegisterWeb(rawURL, title string) string { return "" } if shortSourceAliasRE.MatchString(rawURL) { - alias := strings.ToLower(rawURL) - r.mu.RLock() - _, known := r.webByAlias[alias] - r.mu.RUnlock() - if known { - return alias + return knownHandle(r.webs, rawURL) + } + // Dedup on the canonical (fragment-stripped) URL while decoding back to + // the raw URL the model was originally shown. + return r.webs.register(canonicalWebURL(rawURL), rawURL, webMeta{title: title}, func(dst *webMeta, src webMeta) { + if dst.title == "" && src.title != "" { + dst.title = src.title } - return "" - } - key := canonicalWebURL(rawURL) - r.mu.Lock() - defer r.mu.Unlock() - if existing := r.webByURL[key]; existing != nil { - if existing.Title == "" && title != "" { - existing.Title = title - } - return existing.Alias - } - ref := &WebReference{ - Alias: fmt.Sprintf("w%d", len(r.webByAlias)+1), - URL: rawURL, - Title: title, - } - r.webByURL[key] = ref - r.webByAlias[ref.Alias] = ref - return ref.Alias + }) } func canonicalWebURL(raw string) string { @@ -250,12 +181,8 @@ func firstNonEmpty(values ...string) string { } func (r *sourceRegistry) ChunkAlias(id string) string { - r.mu.RLock() - defer r.mu.RUnlock() - if ref := r.chunkByID[id]; ref != nil { - return ref.Alias - } - return "" + handle, _ := r.chunks.handleForKey(id) + return handle } // toolArgumentPolicy decides whether a source-bearing JSON key belongs to a @@ -601,38 +528,34 @@ func (r *sourceRegistry) walkJSON(key string, value interface{}, encode bool, al } func (r *sourceRegistry) aliasForRealValue(real string) string { - r.mu.RLock() - defer r.mu.RUnlock() - if ref := r.chunkByID[real]; ref != nil { - return ref.Alias + if handle, ok := r.chunks.handleForKey(real); ok { + return handle } - if alias := r.docToAlias[real]; alias != "" { - return alias + if handle, ok := r.docs.handleForKey(real); ok { + return handle } - if alias := r.kbToAlias[real]; alias != "" { - return alias + if handle, ok := r.kbs.handleForKey(real); ok { + return handle } - if ref := r.webByURL[canonicalWebURL(real)]; ref != nil { - return ref.Alias + if handle, ok := r.webs.handleForKey(canonicalWebURL(real)); ok { + return handle } return "" } func (r *sourceRegistry) realForAlias(alias string) string { alias = strings.ToLower(strings.TrimSpace(alias)) - r.mu.RLock() - defer r.mu.RUnlock() - if ref := r.chunkByAlias[alias]; ref != nil { - return ref.ChunkID - } - if real := r.aliasToDoc[alias]; real != "" { + if real, _, ok := r.chunks.resolve(alias); ok { return real } - if real := r.aliasToKB[alias]; real != "" { + if real, _, ok := r.docs.resolve(alias); ok { return real } - if ref := r.webByAlias[alias]; ref != nil { - return ref.URL + if real, _, ok := r.kbs.resolve(alias); ok { + return real + } + if real, _, ok := r.webs.resolve(alias); ok { + return real } return "" } @@ -644,26 +567,17 @@ func (r *sourceRegistry) CompactKnownText(text string) string { if r == nil || text == "" { return text } - type pair struct{ real, alias string } - r.mu.RLock() - pairs := make([]pair, 0, len(r.chunkByID)+len(r.docToAlias)+len(r.kbToAlias)+len(r.webByURL)) - for real, ref := range r.chunkByID { - pairs = append(pairs, pair{real, ref.Alias}) - } - for real, alias := range r.docToAlias { - pairs = append(pairs, pair{real, alias}) - } - for real, alias := range r.kbToAlias { - pairs = append(pairs, pair{real, alias}) - } - for _, ref := range r.webByURL { - pairs = append(pairs, pair{ref.URL, ref.Alias}) - } - r.mu.RUnlock() - sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].real) > len(pairs[j].real) }) + // The snapshot spans all four source tables and is sorted longest-value + // first GLOBALLY: a web URL may contain a registered document UUID as a + // substring, so per-table passes could corrupt the longer value. + pairs := r.chunks.pairs() + pairs = append(pairs, r.docs.pairs()...) + pairs = append(pairs, r.kbs.pairs()...) + pairs = append(pairs, r.webs.pairs()...) + sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].value) > len(pairs[j].value) }) for _, item := range pairs { - if item.real != "" { - text = strings.ReplaceAll(text, item.real, item.alias) + if item.value != "" { + text = strings.ReplaceAll(text, item.value, item.handle) } } return text diff --git a/internal/modelcontext/sources_test.go b/internal/modelcontext/sources_test.go index 3eca204c..c0777f06 100644 --- a/internal/modelcontext/sources_test.go +++ b/internal/modelcontext/sources_test.go @@ -45,10 +45,10 @@ func TestRegisterDoesNotTreatModelAliasesAsNewDurableIdentities(t *testing.T) { require.Equal(t, "w1", r.RegisterWeb("w1", "")) require.Empty(t, r.RegisterDocument("d99")) require.Empty(t, r.RegisterDocument("c1"), "a chunk handle must not be accepted as a document identity") - require.Equal(t, 1, len(r.chunkByAlias)) - require.Equal(t, 1, len(r.aliasToDoc)) - require.Equal(t, 1, len(r.aliasToKB)) - require.Equal(t, 1, len(r.webByAlias)) + require.Equal(t, 1, r.chunks.size()) + require.Equal(t, 1, r.docs.size()) + require.Equal(t, 1, r.kbs.size()) + require.Equal(t, 1, r.webs.size()) } func TestRegistrySuppressesSourceCitationsWhenDisabled(t *testing.T) {