fix(agent): recover tool panics instead of crashing the server
Parallel tool calls run on errgroup goroutines, and errgroup does not carry a panic back to Wait, so one panicking tool (MCP client, sandbox, SDK) took down the whole process. The registry now turns a panic into a failed tool result, and the parallel path guards its own bookkeeping as a backstop.main
parent
8619bdf3c5
commit
ef6f28c1ca
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -315,6 +316,18 @@ func (e *AgentEngine) executeToolCallsParallel(
|
|||
}
|
||||
readCtx := gCtx
|
||||
g.Go(func() error {
|
||||
// The registry recovers panics inside tool execution; this guards
|
||||
// the bookkeeping around it, which would otherwise crash the
|
||||
// process from a goroutine nothing above can recover.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Errorf(ctx, "[Agent][Round-%d] Tool call %s panicked: %v\n%s",
|
||||
round, tc.Function.Name, r, debug.Stack())
|
||||
mu.Lock()
|
||||
results[i] = panickedToolCall(tc, i)
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
toolCall := e.runToolCall(readCtx, tc, i, iteration, round, sessionID, assistantMessageID)
|
||||
mu.Lock()
|
||||
results[i] = toolCall
|
||||
|
|
@ -332,6 +345,21 @@ func (e *AgentEngine) executeToolCallsParallel(
|
|||
}
|
||||
}
|
||||
|
||||
// panickedToolCall is the failed record for a call whose handling panicked, so
|
||||
// the round still returns one result per tool call the model issued.
|
||||
func panickedToolCall(tc types.LLMToolCall, i int) types.ToolCall {
|
||||
return types.ToolCall{
|
||||
ID: agenttools.NormalizeToolCallID(tc.ID, tc.Function.Name, i),
|
||||
Name: tc.Function.Name,
|
||||
Args: map[string]any{"_raw": tc.Function.Arguments},
|
||||
ProviderMetadata: tc.ProviderMetadata,
|
||||
Result: &types.ToolResult{
|
||||
Success: false,
|
||||
Error: fmt.Sprintf("tool %s failed with an internal error", tc.Function.Name),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// emitToolOutcome publishes the result and action events for one finished tool
|
||||
// call. Every path that produces a ToolCall goes through here — sequential,
|
||||
// parallel, and refused-as-truncated — so the UI sees one event shape.
|
||||
|
|
|
|||
|
|
@ -88,3 +88,28 @@ func TestParallelReadsRespectMutationBarriers(t *testing.T) {
|
|||
require.Equal(t, names[i], call.Name, "result events retain original order")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParallelToolPanicFailsOnlyThatCall(t *testing.T) {
|
||||
engine := newTestEngine(t, &mockChat{})
|
||||
engine.toolRegistry = tools.NewToolRegistry()
|
||||
engine.toolRegistry.RegisterTool(&orderedTestTool{
|
||||
BaseTool: tools.NewBaseTool(tools.ToolSearchKnowledge, "", json.RawMessage(`{"type":"object"}`)),
|
||||
run: func(context.Context) *types.ToolResult { panic("tool bug") },
|
||||
})
|
||||
engine.toolRegistry.RegisterTool(&orderedTestTool{
|
||||
BaseTool: tools.NewBaseTool(tools.ToolReadDocument, "", json.RawMessage(`{"type":"object"}`)),
|
||||
run: func(context.Context) *types.ToolResult { return &types.ToolResult{Success: true, Output: "ok"} },
|
||||
})
|
||||
response := &types.ChatResponse{ToolCalls: []types.LLMToolCall{
|
||||
{ID: "a", Function: types.FunctionCall{Name: tools.ToolSearchKnowledge, Arguments: `{}`}},
|
||||
{ID: "b", Function: types.FunctionCall{Name: tools.ToolReadDocument, Arguments: `{}`}},
|
||||
}}
|
||||
step := &types.AgentStep{}
|
||||
|
||||
engine.executeToolCallsParallel(context.Background(), response, step, 0, "session", "message")
|
||||
|
||||
require.Len(t, step.ToolCalls, 2)
|
||||
require.False(t, step.ToolCalls[0].Result.Success)
|
||||
require.Contains(t, step.ToolCalls[0].Result.Error, "internal error")
|
||||
require.True(t, step.ToolCalls[1].Result.Success)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
|
@ -247,7 +248,7 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R
|
|||
maxOutput = toolLimit
|
||||
}
|
||||
}
|
||||
result, execErr := tool.Execute(WithOutputBudget(ctx, maxOutput), args)
|
||||
result, execErr := executeRecovered(WithOutputBudget(ctx, maxOutput), tool, args)
|
||||
if result == nil {
|
||||
result = &types.ToolResult{Success: false, Error: "tool returned no result"}
|
||||
}
|
||||
|
|
@ -290,6 +291,25 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R
|
|||
return result, execErr
|
||||
}
|
||||
|
||||
// executeRecovered runs one tool and turns a panic into a failed result. Tools
|
||||
// run on errgroup goroutines when a round executes in parallel, and errgroup
|
||||
// does not carry a panic back to Wait: an unrecovered one from any tool (MCP
|
||||
// clients, sandboxes, third-party SDKs) would take down the whole server.
|
||||
func executeRecovered(
|
||||
ctx context.Context, tool types.Tool, args json.RawMessage,
|
||||
) (result *types.ToolResult, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Errorf(ctx, "[ToolRegistry] Tool %s panicked: %v\n%s", tool.Name(), r, debug.Stack())
|
||||
// The panic value can carry internals (paths, addresses); the
|
||||
// model and the user only need to know the call did not complete.
|
||||
err = fmt.Errorf("tool %s failed with an internal error", tool.Name())
|
||||
result = &types.ToolResult{Success: false, Error: err.Error()}
|
||||
}
|
||||
}()
|
||||
return tool.Execute(ctx, args)
|
||||
}
|
||||
|
||||
// Cleanup cleans up all registered tools that implement the types.Cleanable interface.
|
||||
// This is called at the end of agent sessions to release tool-specific resources.
|
||||
func (r *ToolRegistry) Cleanup(ctx context.Context) {
|
||||
|
|
|
|||
|
|
@ -61,3 +61,23 @@ func TestRegistryNormalizesMissingAndContradictoryResults(t *testing.T) {
|
|||
require.NotEmpty(t, result.Error)
|
||||
}
|
||||
}
|
||||
|
||||
type panickingTool struct{ BaseTool }
|
||||
|
||||
func (t *panickingTool) Execute(context.Context, json.RawMessage) (*types.ToolResult, error) {
|
||||
var items []int
|
||||
_ = items[len(items)] // index out of range: a runtime panic like a buggy tool would raise
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestRegistryRecoversToolPanic(t *testing.T) {
|
||||
registry := NewToolRegistry()
|
||||
registry.RegisterTool(&panickingTool{BaseTool: NewBaseTool("crashy", "", json.RawMessage(`{"type":"object"}`))})
|
||||
|
||||
result, err := registry.ExecuteTool(context.Background(), "crashy", json.RawMessage(`{}`))
|
||||
require.Error(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.False(t, result.Success)
|
||||
require.Equal(t, "tool crashy failed with an internal error", result.Error)
|
||||
require.NotContains(t, result.Error, "index out of range")
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue