refactor(modelcontext): consolidate stream decoding on one suffix-hold primitive
streamHold captures the shared shape of streaming-safe handle decoding (withhold a trailing byte run that could still become a handle, decode what is released, decide the fate of a held tail at close). The resource decoder, HandleStreamDecoder and the orphan-resource filter become instances whose hold/flush functions are the original logic moved verbatim; the citation stream expander stays a dedicated tag parser. The duplicate res://\d+ regexes merge into resourceHandleShapeRE, all stream types now live in stream.go, and orphan_stream.go is removed.main
parent
213ca0b42e
commit
69b792398e
|
|
@ -93,69 +93,3 @@ func (t *HandleTable) DecodeKnownText(value string) string {
|
|||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// HandleStreamDecoder restores HandleTable values without leaking a handle
|
||||
// split across provider chunks.
|
||||
type HandleStreamDecoder struct {
|
||||
table *HandleTable
|
||||
pending string
|
||||
}
|
||||
|
||||
func NewHandleStreamDecoder(table *HandleTable) *HandleStreamDecoder {
|
||||
return &HandleStreamDecoder{table: table}
|
||||
}
|
||||
|
||||
func (d *HandleStreamDecoder) Feed(chunk string) string {
|
||||
if d == nil || d.table == nil {
|
||||
return chunk
|
||||
}
|
||||
combined := d.pending + chunk
|
||||
d.pending = ""
|
||||
if combined == "" {
|
||||
return ""
|
||||
}
|
||||
start := len(combined)
|
||||
for start > 0 && isHandleTokenByte(combined[start-1]) {
|
||||
start--
|
||||
}
|
||||
tail := combined[start:]
|
||||
// The prefix is immutable after construction, so no lock is needed.
|
||||
prefix := d.table.table.prefix
|
||||
if couldBeNumericHandle(tail, prefix) {
|
||||
d.pending = tail
|
||||
combined = combined[:start]
|
||||
}
|
||||
return d.table.DecodeKnownText(combined)
|
||||
}
|
||||
|
||||
func (d *HandleStreamDecoder) Flush() string {
|
||||
if d == nil || d.table == nil {
|
||||
return ""
|
||||
}
|
||||
tail := d.table.DecodeKnownText(d.pending)
|
||||
d.pending = ""
|
||||
return tail
|
||||
}
|
||||
|
||||
func isHandleTokenByte(value byte) bool {
|
||||
return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' ||
|
||||
value >= '0' && value <= '9' || value == '-' || value == '_'
|
||||
}
|
||||
|
||||
func couldBeNumericHandle(value, prefix string) bool {
|
||||
if value == "" || prefix == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(prefix, value) {
|
||||
return true
|
||||
}
|
||||
if !strings.HasPrefix(value, prefix) || len(value) == len(prefix) {
|
||||
return false
|
||||
}
|
||||
for _, char := range value[len(prefix):] {
|
||||
if char < '0' || char > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
package modelcontext
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var orphanResourceHandleRE = regexp.MustCompile(`res://\d+`)
|
||||
|
||||
// orphanResourceStreamFilter removes unresolved resource handles only after
|
||||
// the registered resource decoder has had a chance to restore known ones. It
|
||||
// buffers a possible handle suffix so provider chunk boundaries cannot leak a
|
||||
// partial internal token to the UI.
|
||||
type orphanResourceStreamFilter struct {
|
||||
pending string
|
||||
}
|
||||
|
||||
func (f *orphanResourceStreamFilter) Feed(chunk string) string {
|
||||
if f == nil {
|
||||
return chunk
|
||||
}
|
||||
combined := f.pending + chunk
|
||||
f.pending = ""
|
||||
if combined == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
const prefix = "res://"
|
||||
holdAt := -1
|
||||
// Unknown handles must be safe across every provider split, including
|
||||
// "re" + "s://9999". This may defer at most three ordinary characters
|
||||
// until the next chunk; Flush preserves them when they are normal prose.
|
||||
for n := 1; n < len(prefix); n++ {
|
||||
if strings.HasSuffix(combined, prefix[:n]) {
|
||||
holdAt = len(combined) - n
|
||||
}
|
||||
}
|
||||
if idx := strings.LastIndex(combined, prefix); idx >= 0 {
|
||||
suffix := combined[idx+len(prefix):]
|
||||
if suffix == "" || allDigits(suffix) {
|
||||
holdAt = idx
|
||||
}
|
||||
}
|
||||
if holdAt >= 0 {
|
||||
f.pending = combined[holdAt:]
|
||||
combined = combined[:holdAt]
|
||||
}
|
||||
return orphanResourceHandleRE.ReplaceAllString(combined, "")
|
||||
}
|
||||
|
||||
func (f *orphanResourceStreamFilter) Flush() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
pending := f.pending
|
||||
f.pending = ""
|
||||
// A stream that ends mid-token must not surface the model-context
|
||||
// protocol fragment. Preserve ordinary r/re/res prose, but discard any
|
||||
// suffix that has already crossed into the reserved URL-like syntax.
|
||||
if strings.HasPrefix("res://", pending) && len(pending) >= len("res:") {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(pending, "res://") {
|
||||
digits := strings.TrimPrefix(pending, "res://")
|
||||
if digits == "" || allDigits(digits) {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
pending = orphanResourceHandleRE.ReplaceAllString(pending, "")
|
||||
return pending
|
||||
}
|
||||
|
||||
func allDigits(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if char < '0' || char > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
@ -171,7 +171,7 @@ func (r *Registry) StreamDecoder() *StreamDecoder {
|
|||
resources: newResourceStreamDecoder(r.resources),
|
||||
sources: newCitationStreamExpander(r.sources),
|
||||
issues: NewHandleStreamDecoder(r.issues),
|
||||
orphans: &orphanResourceStreamFilter{},
|
||||
orphans: newOrphanResourceStreamFilter(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -289,51 +289,3 @@ func (r *Registry) DecodeOutputText(text string) string {
|
|||
text = r.sources.ExpandText(text)
|
||||
return r.issues.DecodeKnownText(text)
|
||||
}
|
||||
|
||||
// StreamDecoder composes resource restoration and source-citation expansion
|
||||
// so callers cannot split handle processing or apply stages in the wrong order.
|
||||
type StreamDecoder struct {
|
||||
resources *resourceStreamDecoder
|
||||
sources *citationStreamExpander
|
||||
issues *HandleStreamDecoder
|
||||
orphans *orphanResourceStreamFilter
|
||||
}
|
||||
|
||||
func (d *StreamDecoder) Feed(chunk string) string {
|
||||
if d == nil {
|
||||
return chunk
|
||||
}
|
||||
if d.resources != nil {
|
||||
chunk = d.resources.Feed(chunk)
|
||||
}
|
||||
if d.sources != nil {
|
||||
chunk = d.sources.Feed(chunk)
|
||||
}
|
||||
if d.issues != nil {
|
||||
chunk = d.issues.Feed(chunk)
|
||||
}
|
||||
if d.orphans != nil {
|
||||
chunk = d.orphans.Feed(chunk)
|
||||
}
|
||||
return chunk
|
||||
}
|
||||
|
||||
func (d *StreamDecoder) Flush() string {
|
||||
if d == nil {
|
||||
return ""
|
||||
}
|
||||
var tail string
|
||||
if d.resources != nil {
|
||||
tail = d.resources.Flush()
|
||||
}
|
||||
if d.sources != nil {
|
||||
tail = d.sources.Feed(tail) + d.sources.Flush()
|
||||
}
|
||||
if d.issues != nil {
|
||||
tail = d.issues.Feed(tail) + d.issues.Flush()
|
||||
}
|
||||
if d.orphans != nil {
|
||||
tail = d.orphans.Feed(tail) + d.orphans.Flush()
|
||||
}
|
||||
return tail
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,10 +34,11 @@ var storedRefRE = regexp.MustCompile(
|
|||
`summary/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`,
|
||||
)
|
||||
|
||||
// aliasShapeRE matches the alias syntax produced by EncodeText. It is used only
|
||||
// to spot alias-shaped tokens the model emitted that the registry cannot map
|
||||
// back — either a hallucinated reference or a coincidental collision.
|
||||
var aliasShapeRE = regexp.MustCompile(`res://\d+`)
|
||||
// resourceHandleShapeRE matches the handle syntax produced by EncodeText. It
|
||||
// is used to spot handle-shaped tokens the registry cannot map back — either a
|
||||
// hallucinated reference or a coincidental collision — and by the stream-side
|
||||
// orphan filter.
|
||||
var resourceHandleShapeRE = regexp.MustCompile(`res://\d+`)
|
||||
|
||||
// resourceRegistry assigns low-entropy, request-local aliases to stable resource
|
||||
// handles. It is safe to reuse across all rounds of one Agent execution.
|
||||
|
|
@ -85,7 +86,7 @@ func (r *resourceRegistry) StripOrphanAliases(value string) string {
|
|||
if value == "" {
|
||||
return value
|
||||
}
|
||||
return aliasShapeRE.ReplaceAllString(value, "")
|
||||
return resourceHandleShapeRE.ReplaceAllString(value, "")
|
||||
}
|
||||
|
||||
// EncodeMessages returns a copied message slice with textual references
|
||||
|
|
@ -133,7 +134,7 @@ func (r *resourceRegistry) OrphanAliases(decoded string) []string {
|
|||
}
|
||||
var orphans []string
|
||||
seen := make(map[string]struct{})
|
||||
for _, match := range aliasShapeRE.FindAllString(decoded, -1) {
|
||||
for _, match := range resourceHandleShapeRE.FindAllString(decoded, -1) {
|
||||
if r != nil && r.table.has(match) {
|
||||
continue
|
||||
}
|
||||
|
|
@ -154,51 +155,3 @@ func (r *resourceRegistry) aliases() []string {
|
|||
}
|
||||
return aliases
|
||||
}
|
||||
|
||||
// resourceStreamDecoder holds only a suffix that could be the beginning of a known
|
||||
// alias, allowing aliases split across provider chunks to round-trip exactly.
|
||||
type resourceStreamDecoder struct {
|
||||
registry *resourceRegistry
|
||||
pending string
|
||||
}
|
||||
|
||||
// newResourceStreamDecoder creates an alias decoder for one streaming text channel.
|
||||
func newResourceStreamDecoder(registry *resourceRegistry) *resourceStreamDecoder {
|
||||
return &resourceStreamDecoder{registry: registry}
|
||||
}
|
||||
|
||||
// Feed decodes a chunk while retaining suffixes that may complete an alias in
|
||||
// the next provider chunk.
|
||||
func (d *resourceStreamDecoder) Feed(chunk string) string {
|
||||
if d == nil || d.registry == nil {
|
||||
return chunk
|
||||
}
|
||||
combined := d.pending + chunk
|
||||
d.pending = ""
|
||||
hold := 0
|
||||
for _, alias := range d.registry.aliases() {
|
||||
// A provider may split the token at any byte boundary, including
|
||||
// "re" + "s://0001". Holding at most the short matching suffix is the
|
||||
// only way to guarantee that a request-local handle never leaks.
|
||||
for n := 1; n < len(alias); n++ {
|
||||
if n > hold && strings.HasSuffix(combined, alias[:n]) {
|
||||
hold = n
|
||||
}
|
||||
}
|
||||
}
|
||||
if hold > 0 {
|
||||
d.pending = combined[len(combined)-hold:]
|
||||
combined = combined[:len(combined)-hold]
|
||||
}
|
||||
return d.registry.DecodeText(combined)
|
||||
}
|
||||
|
||||
// Flush returns any buffered suffix when the provider stream closes.
|
||||
func (d *resourceStreamDecoder) Flush() string {
|
||||
if d == nil {
|
||||
return ""
|
||||
}
|
||||
pending := d.pending
|
||||
d.pending = ""
|
||||
return d.registry.DecodeText(pending)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,275 @@
|
|||
// stream.go owns streaming-safe handle decoding: the shared suffix-hold
|
||||
// primitive, the per-space decoders built on it, and the composed public
|
||||
// StreamDecoder that applies every stage in the only safe order.
|
||||
package modelcontext
|
||||
|
||||
import "strings"
|
||||
|
||||
// streamHold is the shared primitive for decoders that must never emit a
|
||||
// partial model handle: each Feed withholds a trailing byte run that could
|
||||
// still grow into a handle in the next provider chunk, and applies the
|
||||
// space-specific decode to everything released. flush decides what happens to
|
||||
// a suffix still held when the stream closes.
|
||||
type streamHold struct {
|
||||
pending string
|
||||
holdLen func(combined string) int // trailing bytes to withhold
|
||||
emit func(released string) string // decode applied to released text
|
||||
flush func(pending string) string // end-of-stream disposition
|
||||
}
|
||||
|
||||
func (h *streamHold) Feed(chunk string) string {
|
||||
combined := h.pending + chunk
|
||||
h.pending = ""
|
||||
if combined == "" {
|
||||
return ""
|
||||
}
|
||||
if hold := h.holdLen(combined); hold > 0 && hold <= len(combined) {
|
||||
h.pending = combined[len(combined)-hold:]
|
||||
combined = combined[:len(combined)-hold]
|
||||
}
|
||||
return h.emit(combined)
|
||||
}
|
||||
|
||||
func (h *streamHold) Flush() string {
|
||||
pending := h.pending
|
||||
h.pending = ""
|
||||
return h.flush(pending)
|
||||
}
|
||||
|
||||
// resourceStreamDecoder restores res:// handles split across provider chunks.
|
||||
type resourceStreamDecoder struct {
|
||||
hold streamHold
|
||||
}
|
||||
|
||||
func newResourceStreamDecoder(registry *resourceRegistry) *resourceStreamDecoder {
|
||||
return &resourceStreamDecoder{hold: streamHold{
|
||||
holdLen: func(combined string) int {
|
||||
hold := 0
|
||||
for _, alias := range registry.aliases() {
|
||||
// A provider may split the token at any byte boundary, including
|
||||
// "re" + "s://0001". Holding at most the short matching suffix is
|
||||
// the only way to guarantee a request-local handle never leaks.
|
||||
for n := 1; n < len(alias); n++ {
|
||||
if n > hold && strings.HasSuffix(combined, alias[:n]) {
|
||||
hold = n
|
||||
}
|
||||
}
|
||||
}
|
||||
return hold
|
||||
},
|
||||
emit: registry.DecodeText,
|
||||
flush: registry.DecodeText,
|
||||
}}
|
||||
}
|
||||
|
||||
func (d *resourceStreamDecoder) Feed(chunk string) string {
|
||||
if d == nil {
|
||||
return chunk
|
||||
}
|
||||
return d.hold.Feed(chunk)
|
||||
}
|
||||
|
||||
func (d *resourceStreamDecoder) Flush() string {
|
||||
if d == nil {
|
||||
return ""
|
||||
}
|
||||
return d.hold.Flush()
|
||||
}
|
||||
|
||||
// HandleStreamDecoder restores HandleTable values without leaking a handle
|
||||
// split across provider chunks.
|
||||
type HandleStreamDecoder struct {
|
||||
table *HandleTable
|
||||
hold streamHold
|
||||
}
|
||||
|
||||
func NewHandleStreamDecoder(table *HandleTable) *HandleStreamDecoder {
|
||||
return &HandleStreamDecoder{table: table, hold: streamHold{
|
||||
holdLen: func(combined string) int {
|
||||
start := len(combined)
|
||||
for start > 0 && isHandleTokenByte(combined[start-1]) {
|
||||
start--
|
||||
}
|
||||
tail := combined[start:]
|
||||
// The prefix is immutable after construction, so no lock is needed.
|
||||
if couldBeNumericHandle(tail, table.table.prefix) {
|
||||
return len(tail)
|
||||
}
|
||||
return 0
|
||||
},
|
||||
emit: table.DecodeKnownText,
|
||||
flush: table.DecodeKnownText,
|
||||
}}
|
||||
}
|
||||
|
||||
func (d *HandleStreamDecoder) Feed(chunk string) string {
|
||||
if d == nil || d.table == nil {
|
||||
return chunk
|
||||
}
|
||||
return d.hold.Feed(chunk)
|
||||
}
|
||||
|
||||
func (d *HandleStreamDecoder) Flush() string {
|
||||
if d == nil || d.table == nil {
|
||||
return ""
|
||||
}
|
||||
return d.hold.Flush()
|
||||
}
|
||||
|
||||
func isHandleTokenByte(value byte) bool {
|
||||
return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' ||
|
||||
value >= '0' && value <= '9' || value == '-' || value == '_'
|
||||
}
|
||||
|
||||
func couldBeNumericHandle(value, prefix string) bool {
|
||||
if value == "" || prefix == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(prefix, value) {
|
||||
return true
|
||||
}
|
||||
if !strings.HasPrefix(value, prefix) || len(value) == len(prefix) {
|
||||
return false
|
||||
}
|
||||
for _, char := range value[len(prefix):] {
|
||||
if char < '0' || char > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// orphanResourceStreamFilter removes unresolved resource handles only after
|
||||
// the registered resource decoder has had a chance to restore known ones. It
|
||||
// buffers a possible handle suffix so provider chunk boundaries cannot leak a
|
||||
// partial internal token to the UI.
|
||||
type orphanResourceStreamFilter struct {
|
||||
hold streamHold
|
||||
}
|
||||
|
||||
func newOrphanResourceStreamFilter() *orphanResourceStreamFilter {
|
||||
return &orphanResourceStreamFilter{hold: streamHold{
|
||||
holdLen: orphanResourceHoldLen,
|
||||
emit: func(released string) string {
|
||||
return resourceHandleShapeRE.ReplaceAllString(released, "")
|
||||
},
|
||||
flush: orphanResourceFlush,
|
||||
}}
|
||||
}
|
||||
|
||||
func orphanResourceHoldLen(combined string) int {
|
||||
const prefix = "res://"
|
||||
holdAt := -1
|
||||
// Unknown handles must be safe across every provider split, including
|
||||
// "re" + "s://9999". This may defer at most a few ordinary characters
|
||||
// until the next chunk; Flush preserves them when they are normal prose.
|
||||
for n := 1; n < len(prefix); n++ {
|
||||
if strings.HasSuffix(combined, prefix[:n]) {
|
||||
holdAt = len(combined) - n
|
||||
}
|
||||
}
|
||||
if idx := strings.LastIndex(combined, prefix); idx >= 0 {
|
||||
suffix := combined[idx+len(prefix):]
|
||||
if suffix == "" || allDigits(suffix) {
|
||||
holdAt = idx
|
||||
}
|
||||
}
|
||||
if holdAt < 0 {
|
||||
return 0
|
||||
}
|
||||
return len(combined) - holdAt
|
||||
}
|
||||
|
||||
func orphanResourceFlush(pending string) string {
|
||||
// A stream that ends mid-token must not surface the model-context
|
||||
// protocol fragment. Preserve ordinary r/re/res prose, but discard any
|
||||
// suffix that has already crossed into the reserved URL-like syntax.
|
||||
if strings.HasPrefix("res://", pending) && len(pending) >= len("res:") {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(pending, "res://") {
|
||||
digits := strings.TrimPrefix(pending, "res://")
|
||||
if digits == "" || allDigits(digits) {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return resourceHandleShapeRE.ReplaceAllString(pending, "")
|
||||
}
|
||||
|
||||
func (f *orphanResourceStreamFilter) Feed(chunk string) string {
|
||||
if f == nil {
|
||||
return chunk
|
||||
}
|
||||
return f.hold.Feed(chunk)
|
||||
}
|
||||
|
||||
func (f *orphanResourceStreamFilter) Flush() string {
|
||||
if f == nil {
|
||||
return ""
|
||||
}
|
||||
return f.hold.Flush()
|
||||
}
|
||||
|
||||
func allDigits(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, char := range value {
|
||||
if char < '0' || char > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// StreamDecoder composes resource restoration, source-citation expansion,
|
||||
// issue-handle decoding and orphan filtering so callers cannot split handle
|
||||
// processing or apply stages in the wrong order.
|
||||
type StreamDecoder struct {
|
||||
resources *resourceStreamDecoder
|
||||
sources *citationStreamExpander
|
||||
issues *HandleStreamDecoder
|
||||
orphans *orphanResourceStreamFilter
|
||||
}
|
||||
|
||||
func (d *StreamDecoder) Feed(chunk string) string {
|
||||
if d == nil {
|
||||
return chunk
|
||||
}
|
||||
if d.resources != nil {
|
||||
chunk = d.resources.Feed(chunk)
|
||||
}
|
||||
if d.sources != nil {
|
||||
chunk = d.sources.Feed(chunk)
|
||||
}
|
||||
if d.issues != nil {
|
||||
chunk = d.issues.Feed(chunk)
|
||||
}
|
||||
if d.orphans != nil {
|
||||
chunk = d.orphans.Feed(chunk)
|
||||
}
|
||||
return chunk
|
||||
}
|
||||
|
||||
func (d *StreamDecoder) Flush() string {
|
||||
if d == nil {
|
||||
return ""
|
||||
}
|
||||
// Each stage's tail must be fed THROUGH the later stages before those
|
||||
// stages flush their own pending suffix, otherwise a handle completed by
|
||||
// an earlier stage's tail would bypass later decoding.
|
||||
var tail string
|
||||
if d.resources != nil {
|
||||
tail = d.resources.Flush()
|
||||
}
|
||||
if d.sources != nil {
|
||||
tail = d.sources.Feed(tail) + d.sources.Flush()
|
||||
}
|
||||
if d.issues != nil {
|
||||
tail = d.issues.Feed(tail) + d.issues.Flush()
|
||||
}
|
||||
if d.orphans != nil {
|
||||
tail = d.orphans.Feed(tail) + d.orphans.Flush()
|
||||
}
|
||||
return tail
|
||||
}
|
||||
Loading…
Reference in New Issue