refactor(modelcontext): rebuild every handle space on one generic table
handleTable[M] is the single bidirectional primitive behind all model handles: source chunks/documents/KBs/web pages, res:// resource handles, and the exported HandleTable (iN issue handles, ingest ref-N/c000 handles). Metadata lives under the same lock as the mapping, web references dedup on the canonical URL while decoding to the raw URL, and the append-only counter preserves historical numbering exactly. Behavior is unchanged and proven by the moved test suites; the per-space text-rewrite strategies (plain substring for resources, word-boundary regex for HandleTable, shape-regex lookup for sources) deliberately stay in their wrappers because each is load-bearing wire behavior. The unused ChunkReference.Alias field is dropped — handles are derived from the table, never stored in metadata.main
parent
286eadac0a
commit
213ca0b42e
|
|
@ -172,19 +172,15 @@ func (r *sourceRegistry) ExpandText(text string) string {
|
|||
return ""
|
||||
}
|
||||
alias := strings.ToLower(match[1])
|
||||
r.mu.RLock()
|
||||
chunkRef := r.chunkByAlias[alias]
|
||||
webRef := r.webByAlias[alias]
|
||||
r.mu.RUnlock()
|
||||
if chunkRef != nil {
|
||||
attrs := fmt.Sprintf(`doc="%s" chunk_id="%s"`, escapeAttr(chunkRef.DocumentTitle), escapeAttr(chunkRef.ChunkID))
|
||||
if chunkID, chunkRef, ok := r.chunks.resolve(alias); ok {
|
||||
attrs := fmt.Sprintf(`doc="%s" chunk_id="%s"`, escapeAttr(chunkRef.DocumentTitle), escapeAttr(chunkID))
|
||||
if chunkRef.KnowledgeBaseID != "" {
|
||||
attrs += fmt.Sprintf(` kb_id="%s"`, escapeAttr(chunkRef.KnowledgeBaseID))
|
||||
}
|
||||
return "<kb " + attrs + " />"
|
||||
}
|
||||
if webRef != nil {
|
||||
return fmt.Sprintf(`<web url="%s" title="%s" />`, escapeAttr(webRef.URL), escapeAttr(webRef.Title))
|
||||
if rawURL, web, ok := r.webs.resolve(alias); ok {
|
||||
return fmt.Sprintf(`<web url="%s" title="%s" />`, escapeAttr(rawURL), escapeAttr(web.title))
|
||||
}
|
||||
return ""
|
||||
})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,137 @@
|
|||
package modelcontext
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// handleTable is the single bidirectional primitive behind every model-handle
|
||||
// space: cN/dN/bN/wN source handles, iN issue handles, res://NNNN resource
|
||||
// handles, and the ref-N / c000 ingest handles exposed through HandleTable.
|
||||
//
|
||||
// It maps a durable dedup key to a sequentially allocated handle and stores
|
||||
// the durable value plus optional metadata under one lock, so a resolvable
|
||||
// handle can never be observed without its metadata. The key and the value
|
||||
// may differ (web references dedup on a canonicalized URL but decode back to
|
||||
// the original raw URL). Entries are never removed, which makes the counter
|
||||
// equivalent to the historical len(map)+1 allocation and keeps handle
|
||||
// numbering stable for the lifetime of the table.
|
||||
type handleTable[M any] struct {
|
||||
mu sync.RWMutex
|
||||
prefix string
|
||||
width int
|
||||
next int
|
||||
handleByKey map[string]string
|
||||
entryByHandle map[string]*handleEntry[M]
|
||||
}
|
||||
|
||||
type handleEntry[M any] struct {
|
||||
value string // durable value a handle decodes back to
|
||||
meta M
|
||||
}
|
||||
|
||||
// handlePair is a snapshot row for text codecs (compaction/decoding).
|
||||
type handlePair struct {
|
||||
value string
|
||||
handle string
|
||||
}
|
||||
|
||||
// newHandleTable creates a handle space such as c1 (prefix="c", width=0,
|
||||
// start=1), c000 (prefix="c", width=3, start=0) or res://0001
|
||||
// (prefix="res://", width=4, start=1).
|
||||
func newHandleTable[M any](prefix string, width, start int) *handleTable[M] {
|
||||
return &handleTable[M]{
|
||||
prefix: prefix,
|
||||
width: width,
|
||||
next: start,
|
||||
handleByKey: make(map[string]string),
|
||||
entryByHandle: make(map[string]*handleEntry[M]),
|
||||
}
|
||||
}
|
||||
|
||||
// register returns the handle assigned to key, allocating the next one on
|
||||
// first use. value is what the handle decodes back to; merge, when non-nil,
|
||||
// folds the metadata of a repeated registration into the existing entry.
|
||||
func (t *handleTable[M]) register(key, value string, meta M, merge func(dst *M, src M)) string {
|
||||
if t == nil || key == "" {
|
||||
return ""
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if handle, ok := t.handleByKey[key]; ok {
|
||||
if merge != nil {
|
||||
merge(&t.entryByHandle[handle].meta, meta)
|
||||
}
|
||||
return handle
|
||||
}
|
||||
number := fmt.Sprintf("%d", t.next)
|
||||
if t.width > 0 {
|
||||
number = fmt.Sprintf("%0*d", t.width, t.next)
|
||||
}
|
||||
t.next++
|
||||
handle := t.prefix + number
|
||||
t.handleByKey[key] = handle
|
||||
t.entryByHandle[handle] = &handleEntry[M]{value: value, meta: meta}
|
||||
return handle
|
||||
}
|
||||
|
||||
// handleForKey returns an existing handle without allocating one.
|
||||
func (t *handleTable[M]) handleForKey(key string) (string, bool) {
|
||||
if t == nil {
|
||||
return "", false
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
handle, ok := t.handleByKey[key]
|
||||
return handle, ok
|
||||
}
|
||||
|
||||
// resolve returns the durable value and metadata snapshot for a known handle.
|
||||
func (t *handleTable[M]) resolve(handle string) (string, M, bool) {
|
||||
var zero M
|
||||
if t == nil {
|
||||
return "", zero, false
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
entry, ok := t.entryByHandle[handle]
|
||||
if !ok {
|
||||
return "", zero, false
|
||||
}
|
||||
return entry.value, entry.meta, true
|
||||
}
|
||||
|
||||
// has reports whether handle exists in this table.
|
||||
func (t *handleTable[M]) has(handle string) bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
_, ok := t.entryByHandle[handle]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (t *handleTable[M]) size() int {
|
||||
if t == nil {
|
||||
return 0
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return len(t.entryByHandle)
|
||||
}
|
||||
|
||||
// pairs returns a value/handle snapshot for text codecs. Callers own ordering
|
||||
// (e.g. longest-value-first for substring compaction).
|
||||
func (t *handleTable[M]) pairs() []handlePair {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
out := make([]handlePair, 0, len(t.entryByHandle))
|
||||
for handle, entry := range t.entryByHandle {
|
||||
out = append(out, handlePair{value: entry.value, handle: handle})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -1,35 +1,24 @@
|
|||
package modelcontext
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// HandleTable is a typed, invocation-local bidirectional mapping used when a
|
||||
// prompt needs compact handles for durable values. Handles must never be
|
||||
// persisted; Resolve converts model output back to the durable value first.
|
||||
// prompt needs compact handles for durable values (wiki issue iN handles,
|
||||
// ingest ref-N slug handles, c000 citation-batch handles). Handles must never
|
||||
// be persisted; Resolve converts model output back to the durable value first.
|
||||
// It is the exported word-boundary-aware wrapper over handleTable.
|
||||
type HandleTable struct {
|
||||
mu sync.RWMutex
|
||||
prefix string
|
||||
width int
|
||||
next int
|
||||
toHandle map[string]string
|
||||
toValue map[string]string
|
||||
table *handleTable[struct{}]
|
||||
}
|
||||
|
||||
// NewHandleTable creates a handle space such as c000 (prefix=c, width=3,
|
||||
// start=0) or ref-1 (prefix=ref-, width=0, start=1).
|
||||
func NewHandleTable(prefix string, width, start int) *HandleTable {
|
||||
return &HandleTable{
|
||||
prefix: prefix,
|
||||
width: width,
|
||||
next: start,
|
||||
toHandle: make(map[string]string),
|
||||
toValue: make(map[string]string),
|
||||
}
|
||||
return &HandleTable{table: newHandleTable[struct{}](prefix, width, start)}
|
||||
}
|
||||
|
||||
// Register returns the stable handle assigned to value in this table.
|
||||
|
|
@ -41,20 +30,7 @@ func (t *HandleTable) Register(value string) string {
|
|||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if handle := t.toHandle[value]; handle != "" {
|
||||
return handle
|
||||
}
|
||||
number := fmt.Sprintf("%d", t.next)
|
||||
if t.width > 0 {
|
||||
number = fmt.Sprintf("%0*d", t.width, t.next)
|
||||
}
|
||||
t.next++
|
||||
handle := t.prefix + number
|
||||
t.toHandle[value] = handle
|
||||
t.toValue[handle] = value
|
||||
return handle
|
||||
return t.table.register(value, value, struct{}{}, nil)
|
||||
}
|
||||
|
||||
// Handle returns an already registered handle without creating one.
|
||||
|
|
@ -62,10 +38,7 @@ func (t *HandleTable) Handle(value string) (string, bool) {
|
|||
if t == nil {
|
||||
return "", false
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
handle, ok := t.toHandle[value]
|
||||
return handle, ok
|
||||
return t.table.handleForKey(value)
|
||||
}
|
||||
|
||||
// Resolve converts a known model handle back to its durable value.
|
||||
|
|
@ -73,28 +46,19 @@ func (t *HandleTable) Resolve(handle string) (string, bool) {
|
|||
if t == nil {
|
||||
return "", false
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
value, ok := t.toValue[strings.TrimSpace(handle)]
|
||||
value, _, ok := t.table.resolve(strings.TrimSpace(handle))
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func (t *HandleTable) Empty() bool {
|
||||
if t == nil {
|
||||
return true
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return len(t.toHandle) == 0
|
||||
return t == nil || t.table.size() == 0
|
||||
}
|
||||
|
||||
func (t *HandleTable) Len() int {
|
||||
if t == nil {
|
||||
return 0
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return len(t.toHandle)
|
||||
return t.table.size()
|
||||
}
|
||||
|
||||
// EncodeKnownText replaces already-registered durable values with their model
|
||||
|
|
@ -103,13 +67,7 @@ func (t *HandleTable) EncodeKnownText(value string) string {
|
|||
if t == nil || value == "" {
|
||||
return value
|
||||
}
|
||||
t.mu.RLock()
|
||||
type pair struct{ value, handle string }
|
||||
pairs := make([]pair, 0, len(t.toHandle))
|
||||
for durable, handle := range t.toHandle {
|
||||
pairs = append(pairs, pair{durable, handle})
|
||||
}
|
||||
t.mu.RUnlock()
|
||||
pairs := t.table.pairs()
|
||||
sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].value) > len(pairs[j].value) })
|
||||
for _, item := range pairs {
|
||||
value = strings.ReplaceAll(value, item.value, item.handle)
|
||||
|
|
@ -117,25 +75,21 @@ func (t *HandleTable) EncodeKnownText(value string) string {
|
|||
return value
|
||||
}
|
||||
|
||||
// DecodeKnownText restores registered handles in complete text.
|
||||
// DecodeKnownText restores registered handles in complete text. Unlike the
|
||||
// resource codec, replacement is word-boundary-aware so short handles such as
|
||||
// i1 cannot fire inside ordinary prose tokens.
|
||||
func (t *HandleTable) DecodeKnownText(value string) string {
|
||||
if t == nil || value == "" {
|
||||
return value
|
||||
}
|
||||
t.mu.RLock()
|
||||
handles := make([]string, 0, len(t.toValue))
|
||||
for handle := range t.toValue {
|
||||
handles = append(handles, handle)
|
||||
}
|
||||
t.mu.RUnlock()
|
||||
sort.SliceStable(handles, func(i, j int) bool { return len(handles[i]) > len(handles[j]) })
|
||||
for _, handle := range handles {
|
||||
pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(handle) + `\b`)
|
||||
if durable, ok := t.Resolve(handle); ok {
|
||||
// A durable value may legally contain '$'. ReplaceAllString would
|
||||
// interpret it as a regexp expansion, so use a literal callback.
|
||||
value = pattern.ReplaceAllStringFunc(value, func(string) string { return durable })
|
||||
}
|
||||
pairs := t.table.pairs()
|
||||
sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].handle) > len(pairs[j].handle) })
|
||||
for _, item := range pairs {
|
||||
pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(item.handle) + `\b`)
|
||||
// A durable value may legally contain '$'. ReplaceAllString would
|
||||
// interpret it as a regexp expansion, so use a literal callback.
|
||||
durable := item.value
|
||||
value = pattern.ReplaceAllStringFunc(value, func(string) string { return durable })
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
|
@ -165,9 +119,8 @@ func (d *HandleStreamDecoder) Feed(chunk string) string {
|
|||
start--
|
||||
}
|
||||
tail := combined[start:]
|
||||
d.table.mu.RLock()
|
||||
prefix := d.table.prefix
|
||||
d.table.mu.RUnlock()
|
||||
// The prefix is immutable after construction, so no lock is needed.
|
||||
prefix := d.table.table.prefix
|
||||
if couldBeNumericHandle(tail, prefix) {
|
||||
d.pending = tail
|
||||
combined = combined[:start]
|
||||
|
|
|
|||
|
|
@ -4,11 +4,9 @@
|
|||
package modelcontext
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
|
|
@ -44,17 +42,15 @@ var aliasShapeRE = regexp.MustCompile(`res://\d+`)
|
|||
// resourceRegistry assigns low-entropy, request-local aliases to stable resource
|
||||
// handles. It is safe to reuse across all rounds of one Agent execution.
|
||||
type resourceRegistry struct {
|
||||
mu sync.RWMutex
|
||||
refToAlias map[string]string
|
||||
aliasToRef map[string]string
|
||||
table *handleTable[struct{}]
|
||||
}
|
||||
|
||||
// newResourceRegistry creates an empty request-local alias registry.
|
||||
func newResourceRegistry() *resourceRegistry {
|
||||
return &resourceRegistry{
|
||||
refToAlias: make(map[string]string),
|
||||
aliasToRef: make(map[string]string),
|
||||
}
|
||||
// A URL-shaped handle (scheme://digits) keeps the token low-entropy while
|
||||
// looking enough like a link that the model reuses it verbatim inside
|
||||
// Markdown image/link syntax instead of reasoning about or rewriting it.
|
||||
return &resourceRegistry{table: newHandleTable[struct{}]("res://", 4, 1)}
|
||||
}
|
||||
|
||||
// EncodeText replaces stored references with compact, stable aliases.
|
||||
|
|
@ -63,38 +59,23 @@ func (r *resourceRegistry) EncodeText(value string) string {
|
|||
return value
|
||||
}
|
||||
return storedRefRE.ReplaceAllStringFunc(value, func(ref string) string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if alias, ok := r.refToAlias[ref]; ok {
|
||||
return alias
|
||||
}
|
||||
// A URL-shaped alias (scheme://digits) keeps the token low-entropy while
|
||||
// looking enough like a link that the model reuses it verbatim inside
|
||||
// Markdown image/link syntax instead of reasoning about or rewriting it.
|
||||
alias := fmt.Sprintf("res://%04d", len(r.aliasToRef)+1)
|
||||
r.refToAlias[ref] = alias
|
||||
r.aliasToRef[alias] = ref
|
||||
return alias
|
||||
return r.table.register(ref, ref, struct{}{}, nil)
|
||||
})
|
||||
}
|
||||
|
||||
// DecodeText restores every alias currently known to the registry.
|
||||
// DecodeText restores every alias currently known to the registry. Aliases are
|
||||
// replaced longest-first with no word-boundary check — this plain substring
|
||||
// behavior is load-bearing for handles adjacent to punctuation in Markdown.
|
||||
func (r *resourceRegistry) DecodeText(value string) string {
|
||||
if r == nil || value == "" {
|
||||
return value
|
||||
}
|
||||
r.mu.RLock()
|
||||
aliases := make([]string, 0, len(r.aliasToRef))
|
||||
for alias := range r.aliasToRef {
|
||||
aliases = append(aliases, alias)
|
||||
pairs := r.table.pairs()
|
||||
sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].handle) > len(pairs[j].handle) })
|
||||
for _, item := range pairs {
|
||||
value = strings.ReplaceAll(value, item.handle, item.value)
|
||||
}
|
||||
sort.SliceStable(aliases, func(i, j int) bool { return len(aliases[i]) > len(aliases[j]) })
|
||||
decoded := value
|
||||
for _, alias := range aliases {
|
||||
decoded = strings.ReplaceAll(decoded, alias, r.aliasToRef[alias])
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
return decoded
|
||||
return value
|
||||
}
|
||||
|
||||
// StripOrphanAliases removes alias-shaped tokens after all known aliases have
|
||||
|
|
@ -153,13 +134,8 @@ func (r *resourceRegistry) OrphanAliases(decoded string) []string {
|
|||
var orphans []string
|
||||
seen := make(map[string]struct{})
|
||||
for _, match := range aliasShapeRE.FindAllString(decoded, -1) {
|
||||
if r != nil {
|
||||
r.mu.RLock()
|
||||
_, known := r.aliasToRef[match]
|
||||
r.mu.RUnlock()
|
||||
if known {
|
||||
continue
|
||||
}
|
||||
if r != nil && r.table.has(match) {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[match]; dup {
|
||||
continue
|
||||
|
|
@ -171,11 +147,10 @@ func (r *resourceRegistry) OrphanAliases(decoded string) []string {
|
|||
}
|
||||
|
||||
func (r *resourceRegistry) aliases() []string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
aliases := make([]string, 0, len(r.aliasToRef))
|
||||
for alias := range r.aliasToRef {
|
||||
aliases = append(aliases, alias)
|
||||
pairs := r.table.pairs()
|
||||
aliases := make([]string, 0, len(pairs))
|
||||
for _, item := range pairs {
|
||||
aliases = append(aliases, item.handle)
|
||||
}
|
||||
return aliases
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,19 +7,16 @@ package modelcontext
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
type ChunkReference struct {
|
||||
Alias string
|
||||
ChunkID string
|
||||
KnowledgeID string
|
||||
KnowledgeBaseID string
|
||||
|
|
@ -28,27 +25,20 @@ type ChunkReference struct {
|
|||
ChunkType string
|
||||
}
|
||||
|
||||
type WebReference struct {
|
||||
Alias string
|
||||
URL string
|
||||
Title string
|
||||
// webMeta is the per-web-page metadata stored next to the raw URL.
|
||||
type webMeta struct {
|
||||
title string
|
||||
}
|
||||
|
||||
// sourceRegistry is scoped to one assistant response (including every Agent tool
|
||||
// round). Aliases are never persisted or accepted across requests.
|
||||
// round). Handles are never persisted or accepted across requests.
|
||||
type sourceRegistry struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
citationsEnabled bool
|
||||
|
||||
chunkByID map[string]*ChunkReference
|
||||
chunkByAlias map[string]*ChunkReference
|
||||
docToAlias map[string]string
|
||||
aliasToDoc map[string]string
|
||||
kbToAlias map[string]string
|
||||
aliasToKB map[string]string
|
||||
webByURL map[string]*WebReference
|
||||
webByAlias map[string]*WebReference
|
||||
chunks *handleTable[ChunkReference]
|
||||
docs *handleTable[struct{}]
|
||||
kbs *handleTable[struct{}]
|
||||
webs *handleTable[webMeta]
|
||||
}
|
||||
|
||||
func newSourceRegistry(citationsEnabled ...bool) *sourceRegistry {
|
||||
|
|
@ -58,14 +48,10 @@ func newSourceRegistry(citationsEnabled ...bool) *sourceRegistry {
|
|||
}
|
||||
return &sourceRegistry{
|
||||
citationsEnabled: enabled,
|
||||
chunkByID: make(map[string]*ChunkReference),
|
||||
chunkByAlias: make(map[string]*ChunkReference),
|
||||
docToAlias: make(map[string]string),
|
||||
aliasToDoc: make(map[string]string),
|
||||
kbToAlias: make(map[string]string),
|
||||
aliasToKB: make(map[string]string),
|
||||
webByURL: make(map[string]*WebReference),
|
||||
webByAlias: make(map[string]*WebReference),
|
||||
chunks: newHandleTable[ChunkReference]("c", 0, 1),
|
||||
docs: newHandleTable[struct{}]("d", 0, 1),
|
||||
kbs: newHandleTable[struct{}]("b", 0, 1),
|
||||
webs: newHandleTable[webMeta]("w", 0, 1),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -73,9 +59,18 @@ func (r *sourceRegistry) Count() int {
|
|||
if r == nil {
|
||||
return 0
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return len(r.chunkByAlias) + len(r.webByAlias)
|
||||
return r.chunks.size() + r.webs.size()
|
||||
}
|
||||
|
||||
// knownHandle implements the shared guard for handle-shaped registration
|
||||
// input: a model-emitted handle is echoed back only when it already exists,
|
||||
// and is never accepted as a new durable identity.
|
||||
func knownHandle[M any](table *handleTable[M], id string) string {
|
||||
handle := strings.ToLower(id)
|
||||
if table.has(handle) {
|
||||
return handle
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (r *sourceRegistry) RegisterChunk(ref ChunkReference) string {
|
||||
|
|
@ -87,26 +82,9 @@ func (r *sourceRegistry) RegisterChunk(ref ChunkReference) string {
|
|||
return ""
|
||||
}
|
||||
if shortSourceAliasRE.MatchString(ref.ChunkID) {
|
||||
alias := strings.ToLower(ref.ChunkID)
|
||||
r.mu.RLock()
|
||||
_, known := r.chunkByAlias[alias]
|
||||
r.mu.RUnlock()
|
||||
if known {
|
||||
return alias
|
||||
}
|
||||
return ""
|
||||
return knownHandle(r.chunks, ref.ChunkID)
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if existing := r.chunkByID[ref.ChunkID]; existing != nil {
|
||||
mergeChunkReference(existing, ref)
|
||||
return existing.Alias
|
||||
}
|
||||
ref.Alias = fmt.Sprintf("c%d", len(r.chunkByAlias)+1)
|
||||
copyRef := ref
|
||||
r.chunkByID[ref.ChunkID] = ©Ref
|
||||
r.chunkByAlias[ref.Alias] = ©Ref
|
||||
return ref.Alias
|
||||
return r.chunks.register(ref.ChunkID, ref.ChunkID, ref, mergeChunkReference)
|
||||
}
|
||||
|
||||
func mergeChunkReference(dst *ChunkReference, src ChunkReference) {
|
||||
|
|
@ -133,24 +111,9 @@ func (r *sourceRegistry) RegisterDocument(id string) string {
|
|||
return ""
|
||||
}
|
||||
if shortSourceAliasRE.MatchString(id) {
|
||||
alias := strings.ToLower(id)
|
||||
r.mu.RLock()
|
||||
_, known := r.aliasToDoc[alias]
|
||||
r.mu.RUnlock()
|
||||
if known {
|
||||
return alias
|
||||
}
|
||||
return ""
|
||||
return knownHandle(r.docs, id)
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if alias := r.docToAlias[id]; alias != "" {
|
||||
return alias
|
||||
}
|
||||
alias := fmt.Sprintf("d%d", len(r.aliasToDoc)+1)
|
||||
r.docToAlias[id] = alias
|
||||
r.aliasToDoc[alias] = id
|
||||
return alias
|
||||
return r.docs.register(id, id, struct{}{}, nil)
|
||||
}
|
||||
|
||||
func (r *sourceRegistry) RegisterKnowledgeBase(id string) string {
|
||||
|
|
@ -159,24 +122,9 @@ func (r *sourceRegistry) RegisterKnowledgeBase(id string) string {
|
|||
return ""
|
||||
}
|
||||
if shortSourceAliasRE.MatchString(id) {
|
||||
alias := strings.ToLower(id)
|
||||
r.mu.RLock()
|
||||
_, known := r.aliasToKB[alias]
|
||||
r.mu.RUnlock()
|
||||
if known {
|
||||
return alias
|
||||
}
|
||||
return ""
|
||||
return knownHandle(r.kbs, id)
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if alias := r.kbToAlias[id]; alias != "" {
|
||||
return alias
|
||||
}
|
||||
alias := fmt.Sprintf("b%d", len(r.aliasToKB)+1)
|
||||
r.kbToAlias[id] = alias
|
||||
r.aliasToKB[alias] = id
|
||||
return alias
|
||||
return r.kbs.register(id, id, struct{}{}, nil)
|
||||
}
|
||||
|
||||
func (r *sourceRegistry) RegisterWeb(rawURL, title string) string {
|
||||
|
|
@ -185,32 +133,15 @@ func (r *sourceRegistry) RegisterWeb(rawURL, title string) string {
|
|||
return ""
|
||||
}
|
||||
if shortSourceAliasRE.MatchString(rawURL) {
|
||||
alias := strings.ToLower(rawURL)
|
||||
r.mu.RLock()
|
||||
_, known := r.webByAlias[alias]
|
||||
r.mu.RUnlock()
|
||||
if known {
|
||||
return alias
|
||||
return knownHandle(r.webs, rawURL)
|
||||
}
|
||||
// Dedup on the canonical (fragment-stripped) URL while decoding back to
|
||||
// the raw URL the model was originally shown.
|
||||
return r.webs.register(canonicalWebURL(rawURL), rawURL, webMeta{title: title}, func(dst *webMeta, src webMeta) {
|
||||
if dst.title == "" && src.title != "" {
|
||||
dst.title = src.title
|
||||
}
|
||||
return ""
|
||||
}
|
||||
key := canonicalWebURL(rawURL)
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if existing := r.webByURL[key]; existing != nil {
|
||||
if existing.Title == "" && title != "" {
|
||||
existing.Title = title
|
||||
}
|
||||
return existing.Alias
|
||||
}
|
||||
ref := &WebReference{
|
||||
Alias: fmt.Sprintf("w%d", len(r.webByAlias)+1),
|
||||
URL: rawURL,
|
||||
Title: title,
|
||||
}
|
||||
r.webByURL[key] = ref
|
||||
r.webByAlias[ref.Alias] = ref
|
||||
return ref.Alias
|
||||
})
|
||||
}
|
||||
|
||||
func canonicalWebURL(raw string) string {
|
||||
|
|
@ -250,12 +181,8 @@ func firstNonEmpty(values ...string) string {
|
|||
}
|
||||
|
||||
func (r *sourceRegistry) ChunkAlias(id string) string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if ref := r.chunkByID[id]; ref != nil {
|
||||
return ref.Alias
|
||||
}
|
||||
return ""
|
||||
handle, _ := r.chunks.handleForKey(id)
|
||||
return handle
|
||||
}
|
||||
|
||||
// toolArgumentPolicy decides whether a source-bearing JSON key belongs to a
|
||||
|
|
@ -601,38 +528,34 @@ func (r *sourceRegistry) walkJSON(key string, value interface{}, encode bool, al
|
|||
}
|
||||
|
||||
func (r *sourceRegistry) aliasForRealValue(real string) string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if ref := r.chunkByID[real]; ref != nil {
|
||||
return ref.Alias
|
||||
if handle, ok := r.chunks.handleForKey(real); ok {
|
||||
return handle
|
||||
}
|
||||
if alias := r.docToAlias[real]; alias != "" {
|
||||
return alias
|
||||
if handle, ok := r.docs.handleForKey(real); ok {
|
||||
return handle
|
||||
}
|
||||
if alias := r.kbToAlias[real]; alias != "" {
|
||||
return alias
|
||||
if handle, ok := r.kbs.handleForKey(real); ok {
|
||||
return handle
|
||||
}
|
||||
if ref := r.webByURL[canonicalWebURL(real)]; ref != nil {
|
||||
return ref.Alias
|
||||
if handle, ok := r.webs.handleForKey(canonicalWebURL(real)); ok {
|
||||
return handle
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (r *sourceRegistry) realForAlias(alias string) string {
|
||||
alias = strings.ToLower(strings.TrimSpace(alias))
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if ref := r.chunkByAlias[alias]; ref != nil {
|
||||
return ref.ChunkID
|
||||
}
|
||||
if real := r.aliasToDoc[alias]; real != "" {
|
||||
if real, _, ok := r.chunks.resolve(alias); ok {
|
||||
return real
|
||||
}
|
||||
if real := r.aliasToKB[alias]; real != "" {
|
||||
if real, _, ok := r.docs.resolve(alias); ok {
|
||||
return real
|
||||
}
|
||||
if ref := r.webByAlias[alias]; ref != nil {
|
||||
return ref.URL
|
||||
if real, _, ok := r.kbs.resolve(alias); ok {
|
||||
return real
|
||||
}
|
||||
if real, _, ok := r.webs.resolve(alias); ok {
|
||||
return real
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
@ -644,26 +567,17 @@ func (r *sourceRegistry) CompactKnownText(text string) string {
|
|||
if r == nil || text == "" {
|
||||
return text
|
||||
}
|
||||
type pair struct{ real, alias string }
|
||||
r.mu.RLock()
|
||||
pairs := make([]pair, 0, len(r.chunkByID)+len(r.docToAlias)+len(r.kbToAlias)+len(r.webByURL))
|
||||
for real, ref := range r.chunkByID {
|
||||
pairs = append(pairs, pair{real, ref.Alias})
|
||||
}
|
||||
for real, alias := range r.docToAlias {
|
||||
pairs = append(pairs, pair{real, alias})
|
||||
}
|
||||
for real, alias := range r.kbToAlias {
|
||||
pairs = append(pairs, pair{real, alias})
|
||||
}
|
||||
for _, ref := range r.webByURL {
|
||||
pairs = append(pairs, pair{ref.URL, ref.Alias})
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].real) > len(pairs[j].real) })
|
||||
// The snapshot spans all four source tables and is sorted longest-value
|
||||
// first GLOBALLY: a web URL may contain a registered document UUID as a
|
||||
// substring, so per-table passes could corrupt the longer value.
|
||||
pairs := r.chunks.pairs()
|
||||
pairs = append(pairs, r.docs.pairs()...)
|
||||
pairs = append(pairs, r.kbs.pairs()...)
|
||||
pairs = append(pairs, r.webs.pairs()...)
|
||||
sort.SliceStable(pairs, func(i, j int) bool { return len(pairs[i].value) > len(pairs[j].value) })
|
||||
for _, item := range pairs {
|
||||
if item.real != "" {
|
||||
text = strings.ReplaceAll(text, item.real, item.alias)
|
||||
if item.value != "" {
|
||||
text = strings.ReplaceAll(text, item.value, item.handle)
|
||||
}
|
||||
}
|
||||
return text
|
||||
|
|
|
|||
|
|
@ -45,10 +45,10 @@ func TestRegisterDoesNotTreatModelAliasesAsNewDurableIdentities(t *testing.T) {
|
|||
require.Equal(t, "w1", r.RegisterWeb("w1", ""))
|
||||
require.Empty(t, r.RegisterDocument("d99"))
|
||||
require.Empty(t, r.RegisterDocument("c1"), "a chunk handle must not be accepted as a document identity")
|
||||
require.Equal(t, 1, len(r.chunkByAlias))
|
||||
require.Equal(t, 1, len(r.aliasToDoc))
|
||||
require.Equal(t, 1, len(r.aliasToKB))
|
||||
require.Equal(t, 1, len(r.webByAlias))
|
||||
require.Equal(t, 1, r.chunks.size())
|
||||
require.Equal(t, 1, r.docs.size())
|
||||
require.Equal(t, 1, r.kbs.size())
|
||||
require.Equal(t, 1, r.webs.size())
|
||||
}
|
||||
|
||||
func TestRegistrySuppressesSourceCitationsWhenDisabled(t *testing.T) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue