fix(agent): recover tool panics instead of crashing the server

Parallel tool calls run on errgroup goroutines, and errgroup does not carry
a panic back to Wait, so one panicking tool (MCP client, sandbox, SDK) took
down the whole process. The registry now turns a panic into a failed tool
result, and the parallel path guards its own bookkeeping as a backstop.
main
wizardchen 2026-09-25 14:44:05 +08:00 committed by lyingbug
parent 8619bdf3c5
commit ef6f28c1ca
4 changed files with 94 additions and 1 deletions

View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"runtime/debug"
"sort"
"sync"
"time"
@ -315,6 +316,18 @@ func (e *AgentEngine) executeToolCallsParallel(
}
readCtx := gCtx
g.Go(func() error {
// The registry recovers panics inside tool execution; this guards
// the bookkeeping around it, which would otherwise crash the
// process from a goroutine nothing above can recover.
defer func() {
if r := recover(); r != nil {
logger.Errorf(ctx, "[Agent][Round-%d] Tool call %s panicked: %v\n%s",
round, tc.Function.Name, r, debug.Stack())
mu.Lock()
results[i] = panickedToolCall(tc, i)
mu.Unlock()
}
}()
toolCall := e.runToolCall(readCtx, tc, i, iteration, round, sessionID, assistantMessageID)
mu.Lock()
results[i] = toolCall
@ -332,6 +345,21 @@ func (e *AgentEngine) executeToolCallsParallel(
}
}
// panickedToolCall is the failed record for a call whose handling panicked, so
// the round still returns one result per tool call the model issued.
func panickedToolCall(tc types.LLMToolCall, i int) types.ToolCall {
return types.ToolCall{
ID: agenttools.NormalizeToolCallID(tc.ID, tc.Function.Name, i),
Name: tc.Function.Name,
Args: map[string]any{"_raw": tc.Function.Arguments},
ProviderMetadata: tc.ProviderMetadata,
Result: &types.ToolResult{
Success: false,
Error: fmt.Sprintf("tool %s failed with an internal error", tc.Function.Name),
},
}
}
// emitToolOutcome publishes the result and action events for one finished tool
// call. Every path that produces a ToolCall goes through here — sequential,
// parallel, and refused-as-truncated — so the UI sees one event shape.

View File

@ -88,3 +88,28 @@ func TestParallelReadsRespectMutationBarriers(t *testing.T) {
require.Equal(t, names[i], call.Name, "result events retain original order")
}
}
func TestParallelToolPanicFailsOnlyThatCall(t *testing.T) {
engine := newTestEngine(t, &mockChat{})
engine.toolRegistry = tools.NewToolRegistry()
engine.toolRegistry.RegisterTool(&orderedTestTool{
BaseTool: tools.NewBaseTool(tools.ToolSearchKnowledge, "", json.RawMessage(`{"type":"object"}`)),
run: func(context.Context) *types.ToolResult { panic("tool bug") },
})
engine.toolRegistry.RegisterTool(&orderedTestTool{
BaseTool: tools.NewBaseTool(tools.ToolReadDocument, "", json.RawMessage(`{"type":"object"}`)),
run: func(context.Context) *types.ToolResult { return &types.ToolResult{Success: true, Output: "ok"} },
})
response := &types.ChatResponse{ToolCalls: []types.LLMToolCall{
{ID: "a", Function: types.FunctionCall{Name: tools.ToolSearchKnowledge, Arguments: `{}`}},
{ID: "b", Function: types.FunctionCall{Name: tools.ToolReadDocument, Arguments: `{}`}},
}}
step := &types.AgentStep{}
engine.executeToolCallsParallel(context.Background(), response, step, 0, "session", "message")
require.Len(t, step.ToolCalls, 2)
require.False(t, step.ToolCalls[0].Result.Success)
require.Contains(t, step.ToolCalls[0].Result.Error, "internal error")
require.True(t, step.ToolCalls[1].Result.Success)
}

View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"runtime/debug"
"sort"
"strings"
"unicode/utf8"
@ -247,7 +248,7 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R
maxOutput = toolLimit
}
}
result, execErr := tool.Execute(WithOutputBudget(ctx, maxOutput), args)
result, execErr := executeRecovered(WithOutputBudget(ctx, maxOutput), tool, args)
if result == nil {
result = &types.ToolResult{Success: false, Error: "tool returned no result"}
}
@ -290,6 +291,25 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R
return result, execErr
}
// executeRecovered runs one tool and turns a panic into a failed result. Tools
// run on errgroup goroutines when a round executes in parallel, and errgroup
// does not carry a panic back to Wait: an unrecovered one from any tool (MCP
// clients, sandboxes, third-party SDKs) would take down the whole server.
func executeRecovered(
ctx context.Context, tool types.Tool, args json.RawMessage,
) (result *types.ToolResult, err error) {
defer func() {
if r := recover(); r != nil {
logger.Errorf(ctx, "[ToolRegistry] Tool %s panicked: %v\n%s", tool.Name(), r, debug.Stack())
// The panic value can carry internals (paths, addresses); the
// model and the user only need to know the call did not complete.
err = fmt.Errorf("tool %s failed with an internal error", tool.Name())
result = &types.ToolResult{Success: false, Error: err.Error()}
}
}()
return tool.Execute(ctx, args)
}
// Cleanup cleans up all registered tools that implement the types.Cleanable interface.
// This is called at the end of agent sessions to release tool-specific resources.
func (r *ToolRegistry) Cleanup(ctx context.Context) {

View File

@ -61,3 +61,23 @@ func TestRegistryNormalizesMissingAndContradictoryResults(t *testing.T) {
require.NotEmpty(t, result.Error)
}
}
type panickingTool struct{ BaseTool }
func (t *panickingTool) Execute(context.Context, json.RawMessage) (*types.ToolResult, error) {
var items []int
_ = items[len(items)] // index out of range: a runtime panic like a buggy tool would raise
return nil, nil
}
func TestRegistryRecoversToolPanic(t *testing.T) {
registry := NewToolRegistry()
registry.RegisterTool(&panickingTool{BaseTool: NewBaseTool("crashy", "", json.RawMessage(`{"type":"object"}`))})
result, err := registry.ExecuteTool(context.Background(), "crashy", json.RawMessage(`{}`))
require.Error(t, err)
require.NotNil(t, result)
require.False(t, result.Success)
require.Equal(t, "tool crashy failed with an internal error", result.Error)
require.NotContains(t, result.Error, "index out of range")
}