From 69b792398e124a80be2c8f95f28f72fa8d284d51 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 27 Jul 2026 01:14:50 +0800 Subject: [PATCH] refactor(modelcontext): consolidate stream decoding on one suffix-hold primitive streamHold captures the shared shape of streaming-safe handle decoding (withhold a trailing byte run that could still become a handle, decode what is released, decide the fate of a held tail at close). The resource decoder, HandleStreamDecoder and the orphan-resource filter become instances whose hold/flush functions are the original logic moved verbatim; the citation stream expander stays a dedicated tag parser. The duplicate res://\d+ regexes merge into resourceHandleShapeRE, all stream types now live in stream.go, and orphan_stream.go is removed. --- internal/modelcontext/handles.go | 66 ------ internal/modelcontext/orphan_stream.go | 83 -------- internal/modelcontext/registry.go | 50 +---- internal/modelcontext/resources.go | 61 +----- internal/modelcontext/stream.go | 275 +++++++++++++++++++++++++ 5 files changed, 283 insertions(+), 252 deletions(-) delete mode 100644 internal/modelcontext/orphan_stream.go create mode 100644 internal/modelcontext/stream.go diff --git a/internal/modelcontext/handles.go b/internal/modelcontext/handles.go index 958caad8..edb63e7f 100644 --- a/internal/modelcontext/handles.go +++ b/internal/modelcontext/handles.go @@ -93,69 +93,3 @@ func (t *HandleTable) DecodeKnownText(value string) string { } return value } - -// HandleStreamDecoder restores HandleTable values without leaking a handle -// split across provider chunks. -type HandleStreamDecoder struct { - table *HandleTable - pending string -} - -func NewHandleStreamDecoder(table *HandleTable) *HandleStreamDecoder { - return &HandleStreamDecoder{table: table} -} - -func (d *HandleStreamDecoder) Feed(chunk string) string { - if d == nil || d.table == nil { - return chunk - } - combined := d.pending + chunk - d.pending = "" - if combined == "" { - return "" - } - start := len(combined) - for start > 0 && isHandleTokenByte(combined[start-1]) { - start-- - } - tail := combined[start:] - // The prefix is immutable after construction, so no lock is needed. - prefix := d.table.table.prefix - if couldBeNumericHandle(tail, prefix) { - d.pending = tail - combined = combined[:start] - } - return d.table.DecodeKnownText(combined) -} - -func (d *HandleStreamDecoder) Flush() string { - if d == nil || d.table == nil { - return "" - } - tail := d.table.DecodeKnownText(d.pending) - d.pending = "" - return tail -} - -func isHandleTokenByte(value byte) bool { - return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' || - value >= '0' && value <= '9' || value == '-' || value == '_' -} - -func couldBeNumericHandle(value, prefix string) bool { - if value == "" || prefix == "" { - return false - } - if strings.HasPrefix(prefix, value) { - return true - } - if !strings.HasPrefix(value, prefix) || len(value) == len(prefix) { - return false - } - for _, char := range value[len(prefix):] { - if char < '0' || char > '9' { - return false - } - } - return true -} diff --git a/internal/modelcontext/orphan_stream.go b/internal/modelcontext/orphan_stream.go deleted file mode 100644 index 424df30b..00000000 --- a/internal/modelcontext/orphan_stream.go +++ /dev/null @@ -1,83 +0,0 @@ -package modelcontext - -import ( - "regexp" - "strings" -) - -var orphanResourceHandleRE = regexp.MustCompile(`res://\d+`) - -// orphanResourceStreamFilter removes unresolved resource handles only after -// the registered resource decoder has had a chance to restore known ones. It -// buffers a possible handle suffix so provider chunk boundaries cannot leak a -// partial internal token to the UI. -type orphanResourceStreamFilter struct { - pending string -} - -func (f *orphanResourceStreamFilter) Feed(chunk string) string { - if f == nil { - return chunk - } - combined := f.pending + chunk - f.pending = "" - if combined == "" { - return "" - } - - const prefix = "res://" - holdAt := -1 - // Unknown handles must be safe across every provider split, including - // "re" + "s://9999". This may defer at most three ordinary characters - // until the next chunk; Flush preserves them when they are normal prose. - for n := 1; n < len(prefix); n++ { - if strings.HasSuffix(combined, prefix[:n]) { - holdAt = len(combined) - n - } - } - if idx := strings.LastIndex(combined, prefix); idx >= 0 { - suffix := combined[idx+len(prefix):] - if suffix == "" || allDigits(suffix) { - holdAt = idx - } - } - if holdAt >= 0 { - f.pending = combined[holdAt:] - combined = combined[:holdAt] - } - return orphanResourceHandleRE.ReplaceAllString(combined, "") -} - -func (f *orphanResourceStreamFilter) Flush() string { - if f == nil { - return "" - } - pending := f.pending - f.pending = "" - // A stream that ends mid-token must not surface the model-context - // protocol fragment. Preserve ordinary r/re/res prose, but discard any - // suffix that has already crossed into the reserved URL-like syntax. - if strings.HasPrefix("res://", pending) && len(pending) >= len("res:") { - return "" - } - if strings.HasPrefix(pending, "res://") { - digits := strings.TrimPrefix(pending, "res://") - if digits == "" || allDigits(digits) { - return "" - } - } - pending = orphanResourceHandleRE.ReplaceAllString(pending, "") - return pending -} - -func allDigits(value string) bool { - if value == "" { - return false - } - for _, char := range value { - if char < '0' || char > '9' { - return false - } - } - return true -} diff --git a/internal/modelcontext/registry.go b/internal/modelcontext/registry.go index b64b6610..604ba343 100644 --- a/internal/modelcontext/registry.go +++ b/internal/modelcontext/registry.go @@ -171,7 +171,7 @@ func (r *Registry) StreamDecoder() *StreamDecoder { resources: newResourceStreamDecoder(r.resources), sources: newCitationStreamExpander(r.sources), issues: NewHandleStreamDecoder(r.issues), - orphans: &orphanResourceStreamFilter{}, + orphans: newOrphanResourceStreamFilter(), } } @@ -289,51 +289,3 @@ func (r *Registry) DecodeOutputText(text string) string { text = r.sources.ExpandText(text) return r.issues.DecodeKnownText(text) } - -// StreamDecoder composes resource restoration and source-citation expansion -// so callers cannot split handle processing or apply stages in the wrong order. -type StreamDecoder struct { - resources *resourceStreamDecoder - sources *citationStreamExpander - issues *HandleStreamDecoder - orphans *orphanResourceStreamFilter -} - -func (d *StreamDecoder) Feed(chunk string) string { - if d == nil { - return chunk - } - if d.resources != nil { - chunk = d.resources.Feed(chunk) - } - if d.sources != nil { - chunk = d.sources.Feed(chunk) - } - if d.issues != nil { - chunk = d.issues.Feed(chunk) - } - if d.orphans != nil { - chunk = d.orphans.Feed(chunk) - } - return chunk -} - -func (d *StreamDecoder) Flush() string { - if d == nil { - return "" - } - var tail string - if d.resources != nil { - tail = d.resources.Flush() - } - if d.sources != nil { - tail = d.sources.Feed(tail) + d.sources.Flush() - } - if d.issues != nil { - tail = d.issues.Feed(tail) + d.issues.Flush() - } - if d.orphans != nil { - tail = d.orphans.Feed(tail) + d.orphans.Flush() - } - return tail -} diff --git a/internal/modelcontext/resources.go b/internal/modelcontext/resources.go index b92402be..8ec77629 100644 --- a/internal/modelcontext/resources.go +++ b/internal/modelcontext/resources.go @@ -34,10 +34,11 @@ var storedRefRE = regexp.MustCompile( `summary/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`, ) -// aliasShapeRE matches the alias syntax produced by EncodeText. It is used only -// to spot alias-shaped tokens the model emitted that the registry cannot map -// back — either a hallucinated reference or a coincidental collision. -var aliasShapeRE = regexp.MustCompile(`res://\d+`) +// resourceHandleShapeRE matches the handle syntax produced by EncodeText. It +// is used to spot handle-shaped tokens the registry cannot map back — either a +// hallucinated reference or a coincidental collision — and by the stream-side +// orphan filter. +var resourceHandleShapeRE = regexp.MustCompile(`res://\d+`) // resourceRegistry assigns low-entropy, request-local aliases to stable resource // handles. It is safe to reuse across all rounds of one Agent execution. @@ -85,7 +86,7 @@ func (r *resourceRegistry) StripOrphanAliases(value string) string { if value == "" { return value } - return aliasShapeRE.ReplaceAllString(value, "") + return resourceHandleShapeRE.ReplaceAllString(value, "") } // EncodeMessages returns a copied message slice with textual references @@ -133,7 +134,7 @@ func (r *resourceRegistry) OrphanAliases(decoded string) []string { } var orphans []string seen := make(map[string]struct{}) - for _, match := range aliasShapeRE.FindAllString(decoded, -1) { + for _, match := range resourceHandleShapeRE.FindAllString(decoded, -1) { if r != nil && r.table.has(match) { continue } @@ -154,51 +155,3 @@ func (r *resourceRegistry) aliases() []string { } return aliases } - -// resourceStreamDecoder holds only a suffix that could be the beginning of a known -// alias, allowing aliases split across provider chunks to round-trip exactly. -type resourceStreamDecoder struct { - registry *resourceRegistry - pending string -} - -// newResourceStreamDecoder creates an alias decoder for one streaming text channel. -func newResourceStreamDecoder(registry *resourceRegistry) *resourceStreamDecoder { - return &resourceStreamDecoder{registry: registry} -} - -// Feed decodes a chunk while retaining suffixes that may complete an alias in -// the next provider chunk. -func (d *resourceStreamDecoder) Feed(chunk string) string { - if d == nil || d.registry == nil { - return chunk - } - combined := d.pending + chunk - d.pending = "" - hold := 0 - for _, alias := range d.registry.aliases() { - // A provider may split the token at any byte boundary, including - // "re" + "s://0001". Holding at most the short matching suffix is the - // only way to guarantee that a request-local handle never leaks. - for n := 1; n < len(alias); n++ { - if n > hold && strings.HasSuffix(combined, alias[:n]) { - hold = n - } - } - } - if hold > 0 { - d.pending = combined[len(combined)-hold:] - combined = combined[:len(combined)-hold] - } - return d.registry.DecodeText(combined) -} - -// Flush returns any buffered suffix when the provider stream closes. -func (d *resourceStreamDecoder) Flush() string { - if d == nil { - return "" - } - pending := d.pending - d.pending = "" - return d.registry.DecodeText(pending) -} diff --git a/internal/modelcontext/stream.go b/internal/modelcontext/stream.go new file mode 100644 index 00000000..3c9ad661 --- /dev/null +++ b/internal/modelcontext/stream.go @@ -0,0 +1,275 @@ +// stream.go owns streaming-safe handle decoding: the shared suffix-hold +// primitive, the per-space decoders built on it, and the composed public +// StreamDecoder that applies every stage in the only safe order. +package modelcontext + +import "strings" + +// streamHold is the shared primitive for decoders that must never emit a +// partial model handle: each Feed withholds a trailing byte run that could +// still grow into a handle in the next provider chunk, and applies the +// space-specific decode to everything released. flush decides what happens to +// a suffix still held when the stream closes. +type streamHold struct { + pending string + holdLen func(combined string) int // trailing bytes to withhold + emit func(released string) string // decode applied to released text + flush func(pending string) string // end-of-stream disposition +} + +func (h *streamHold) Feed(chunk string) string { + combined := h.pending + chunk + h.pending = "" + if combined == "" { + return "" + } + if hold := h.holdLen(combined); hold > 0 && hold <= len(combined) { + h.pending = combined[len(combined)-hold:] + combined = combined[:len(combined)-hold] + } + return h.emit(combined) +} + +func (h *streamHold) Flush() string { + pending := h.pending + h.pending = "" + return h.flush(pending) +} + +// resourceStreamDecoder restores res:// handles split across provider chunks. +type resourceStreamDecoder struct { + hold streamHold +} + +func newResourceStreamDecoder(registry *resourceRegistry) *resourceStreamDecoder { + return &resourceStreamDecoder{hold: streamHold{ + holdLen: func(combined string) int { + hold := 0 + for _, alias := range registry.aliases() { + // A provider may split the token at any byte boundary, including + // "re" + "s://0001". Holding at most the short matching suffix is + // the only way to guarantee a request-local handle never leaks. + for n := 1; n < len(alias); n++ { + if n > hold && strings.HasSuffix(combined, alias[:n]) { + hold = n + } + } + } + return hold + }, + emit: registry.DecodeText, + flush: registry.DecodeText, + }} +} + +func (d *resourceStreamDecoder) Feed(chunk string) string { + if d == nil { + return chunk + } + return d.hold.Feed(chunk) +} + +func (d *resourceStreamDecoder) Flush() string { + if d == nil { + return "" + } + return d.hold.Flush() +} + +// HandleStreamDecoder restores HandleTable values without leaking a handle +// split across provider chunks. +type HandleStreamDecoder struct { + table *HandleTable + hold streamHold +} + +func NewHandleStreamDecoder(table *HandleTable) *HandleStreamDecoder { + return &HandleStreamDecoder{table: table, hold: streamHold{ + holdLen: func(combined string) int { + start := len(combined) + for start > 0 && isHandleTokenByte(combined[start-1]) { + start-- + } + tail := combined[start:] + // The prefix is immutable after construction, so no lock is needed. + if couldBeNumericHandle(tail, table.table.prefix) { + return len(tail) + } + return 0 + }, + emit: table.DecodeKnownText, + flush: table.DecodeKnownText, + }} +} + +func (d *HandleStreamDecoder) Feed(chunk string) string { + if d == nil || d.table == nil { + return chunk + } + return d.hold.Feed(chunk) +} + +func (d *HandleStreamDecoder) Flush() string { + if d == nil || d.table == nil { + return "" + } + return d.hold.Flush() +} + +func isHandleTokenByte(value byte) bool { + return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' || + value >= '0' && value <= '9' || value == '-' || value == '_' +} + +func couldBeNumericHandle(value, prefix string) bool { + if value == "" || prefix == "" { + return false + } + if strings.HasPrefix(prefix, value) { + return true + } + if !strings.HasPrefix(value, prefix) || len(value) == len(prefix) { + return false + } + for _, char := range value[len(prefix):] { + if char < '0' || char > '9' { + return false + } + } + return true +} + +// orphanResourceStreamFilter removes unresolved resource handles only after +// the registered resource decoder has had a chance to restore known ones. It +// buffers a possible handle suffix so provider chunk boundaries cannot leak a +// partial internal token to the UI. +type orphanResourceStreamFilter struct { + hold streamHold +} + +func newOrphanResourceStreamFilter() *orphanResourceStreamFilter { + return &orphanResourceStreamFilter{hold: streamHold{ + holdLen: orphanResourceHoldLen, + emit: func(released string) string { + return resourceHandleShapeRE.ReplaceAllString(released, "") + }, + flush: orphanResourceFlush, + }} +} + +func orphanResourceHoldLen(combined string) int { + const prefix = "res://" + holdAt := -1 + // Unknown handles must be safe across every provider split, including + // "re" + "s://9999". This may defer at most a few ordinary characters + // until the next chunk; Flush preserves them when they are normal prose. + for n := 1; n < len(prefix); n++ { + if strings.HasSuffix(combined, prefix[:n]) { + holdAt = len(combined) - n + } + } + if idx := strings.LastIndex(combined, prefix); idx >= 0 { + suffix := combined[idx+len(prefix):] + if suffix == "" || allDigits(suffix) { + holdAt = idx + } + } + if holdAt < 0 { + return 0 + } + return len(combined) - holdAt +} + +func orphanResourceFlush(pending string) string { + // A stream that ends mid-token must not surface the model-context + // protocol fragment. Preserve ordinary r/re/res prose, but discard any + // suffix that has already crossed into the reserved URL-like syntax. + if strings.HasPrefix("res://", pending) && len(pending) >= len("res:") { + return "" + } + if strings.HasPrefix(pending, "res://") { + digits := strings.TrimPrefix(pending, "res://") + if digits == "" || allDigits(digits) { + return "" + } + } + return resourceHandleShapeRE.ReplaceAllString(pending, "") +} + +func (f *orphanResourceStreamFilter) Feed(chunk string) string { + if f == nil { + return chunk + } + return f.hold.Feed(chunk) +} + +func (f *orphanResourceStreamFilter) Flush() string { + if f == nil { + return "" + } + return f.hold.Flush() +} + +func allDigits(value string) bool { + if value == "" { + return false + } + for _, char := range value { + if char < '0' || char > '9' { + return false + } + } + return true +} + +// StreamDecoder composes resource restoration, source-citation expansion, +// issue-handle decoding and orphan filtering so callers cannot split handle +// processing or apply stages in the wrong order. +type StreamDecoder struct { + resources *resourceStreamDecoder + sources *citationStreamExpander + issues *HandleStreamDecoder + orphans *orphanResourceStreamFilter +} + +func (d *StreamDecoder) Feed(chunk string) string { + if d == nil { + return chunk + } + if d.resources != nil { + chunk = d.resources.Feed(chunk) + } + if d.sources != nil { + chunk = d.sources.Feed(chunk) + } + if d.issues != nil { + chunk = d.issues.Feed(chunk) + } + if d.orphans != nil { + chunk = d.orphans.Feed(chunk) + } + return chunk +} + +func (d *StreamDecoder) Flush() string { + if d == nil { + return "" + } + // Each stage's tail must be fed THROUGH the later stages before those + // stages flush their own pending suffix, otherwise a handle completed by + // an earlier stage's tail would bypass later decoding. + var tail string + if d.resources != nil { + tail = d.resources.Flush() + } + if d.sources != nil { + tail = d.sources.Feed(tail) + d.sources.Flush() + } + if d.issues != nil { + tail = d.issues.Feed(tail) + d.issues.Flush() + } + if d.orphans != nil { + tail = d.orphans.Feed(tail) + d.orphans.Flush() + } + return tail +}