From ef6f28c1cafec72bd05fcb4fbd3feb8144f93e3d Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 25 Sep 2026 14:44:05 +0800 Subject: [PATCH] fix(agent): recover tool panics instead of crashing the server Parallel tool calls run on errgroup goroutines, and errgroup does not carry a panic back to Wait, so one panicking tool (MCP client, sandbox, SDK) took down the whole process. The registry now turns a panic into a failed tool result, and the parallel path guards its own bookkeeping as a backstop. --- internal/agent/act.go | 28 +++++++++++++++++++ internal/agent/act_order_test.go | 25 +++++++++++++++++ internal/agent/tools/registry.go | 22 ++++++++++++++- internal/agent/tools/registry_outcome_test.go | 20 +++++++++++++ 4 files changed, 94 insertions(+), 1 deletion(-) diff --git a/internal/agent/act.go b/internal/agent/act.go index 6933c4fd..3ea6dfb7 100644 --- a/internal/agent/act.go +++ b/internal/agent/act.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "runtime/debug" "sort" "sync" "time" @@ -315,6 +316,18 @@ func (e *AgentEngine) executeToolCallsParallel( } readCtx := gCtx g.Go(func() error { + // The registry recovers panics inside tool execution; this guards + // the bookkeeping around it, which would otherwise crash the + // process from a goroutine nothing above can recover. + defer func() { + if r := recover(); r != nil { + logger.Errorf(ctx, "[Agent][Round-%d] Tool call %s panicked: %v\n%s", + round, tc.Function.Name, r, debug.Stack()) + mu.Lock() + results[i] = panickedToolCall(tc, i) + mu.Unlock() + } + }() toolCall := e.runToolCall(readCtx, tc, i, iteration, round, sessionID, assistantMessageID) mu.Lock() results[i] = toolCall @@ -332,6 +345,21 @@ func (e *AgentEngine) executeToolCallsParallel( } } +// panickedToolCall is the failed record for a call whose handling panicked, so +// the round still returns one result per tool call the model issued. +func panickedToolCall(tc types.LLMToolCall, i int) types.ToolCall { + return types.ToolCall{ + ID: agenttools.NormalizeToolCallID(tc.ID, tc.Function.Name, i), + Name: tc.Function.Name, + Args: map[string]any{"_raw": tc.Function.Arguments}, + ProviderMetadata: tc.ProviderMetadata, + Result: &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("tool %s failed with an internal error", tc.Function.Name), + }, + } +} + // emitToolOutcome publishes the result and action events for one finished tool // call. Every path that produces a ToolCall goes through here — sequential, // parallel, and refused-as-truncated — so the UI sees one event shape. diff --git a/internal/agent/act_order_test.go b/internal/agent/act_order_test.go index 11205c6c..bd6a95b9 100644 --- a/internal/agent/act_order_test.go +++ b/internal/agent/act_order_test.go @@ -88,3 +88,28 @@ func TestParallelReadsRespectMutationBarriers(t *testing.T) { require.Equal(t, names[i], call.Name, "result events retain original order") } } + +func TestParallelToolPanicFailsOnlyThatCall(t *testing.T) { + engine := newTestEngine(t, &mockChat{}) + engine.toolRegistry = tools.NewToolRegistry() + engine.toolRegistry.RegisterTool(&orderedTestTool{ + BaseTool: tools.NewBaseTool(tools.ToolSearchKnowledge, "", json.RawMessage(`{"type":"object"}`)), + run: func(context.Context) *types.ToolResult { panic("tool bug") }, + }) + engine.toolRegistry.RegisterTool(&orderedTestTool{ + BaseTool: tools.NewBaseTool(tools.ToolReadDocument, "", json.RawMessage(`{"type":"object"}`)), + run: func(context.Context) *types.ToolResult { return &types.ToolResult{Success: true, Output: "ok"} }, + }) + response := &types.ChatResponse{ToolCalls: []types.LLMToolCall{ + {ID: "a", Function: types.FunctionCall{Name: tools.ToolSearchKnowledge, Arguments: `{}`}}, + {ID: "b", Function: types.FunctionCall{Name: tools.ToolReadDocument, Arguments: `{}`}}, + }} + step := &types.AgentStep{} + + engine.executeToolCallsParallel(context.Background(), response, step, 0, "session", "message") + + require.Len(t, step.ToolCalls, 2) + require.False(t, step.ToolCalls[0].Result.Success) + require.Contains(t, step.ToolCalls[0].Result.Error, "internal error") + require.True(t, step.ToolCalls[1].Result.Success) +} diff --git a/internal/agent/tools/registry.go b/internal/agent/tools/registry.go index 439f3612..9003e313 100644 --- a/internal/agent/tools/registry.go +++ b/internal/agent/tools/registry.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "runtime/debug" "sort" "strings" "unicode/utf8" @@ -247,7 +248,7 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R maxOutput = toolLimit } } - result, execErr := tool.Execute(WithOutputBudget(ctx, maxOutput), args) + result, execErr := executeRecovered(WithOutputBudget(ctx, maxOutput), tool, args) if result == nil { result = &types.ToolResult{Success: false, Error: "tool returned no result"} } @@ -290,6 +291,25 @@ func (r *ToolRegistry) execute(ctx context.Context, tool types.Tool, args json.R return result, execErr } +// executeRecovered runs one tool and turns a panic into a failed result. Tools +// run on errgroup goroutines when a round executes in parallel, and errgroup +// does not carry a panic back to Wait: an unrecovered one from any tool (MCP +// clients, sandboxes, third-party SDKs) would take down the whole server. +func executeRecovered( + ctx context.Context, tool types.Tool, args json.RawMessage, +) (result *types.ToolResult, err error) { + defer func() { + if r := recover(); r != nil { + logger.Errorf(ctx, "[ToolRegistry] Tool %s panicked: %v\n%s", tool.Name(), r, debug.Stack()) + // The panic value can carry internals (paths, addresses); the + // model and the user only need to know the call did not complete. + err = fmt.Errorf("tool %s failed with an internal error", tool.Name()) + result = &types.ToolResult{Success: false, Error: err.Error()} + } + }() + return tool.Execute(ctx, args) +} + // Cleanup cleans up all registered tools that implement the types.Cleanable interface. // This is called at the end of agent sessions to release tool-specific resources. func (r *ToolRegistry) Cleanup(ctx context.Context) { diff --git a/internal/agent/tools/registry_outcome_test.go b/internal/agent/tools/registry_outcome_test.go index 25ad7b9c..710836cc 100644 --- a/internal/agent/tools/registry_outcome_test.go +++ b/internal/agent/tools/registry_outcome_test.go @@ -61,3 +61,23 @@ func TestRegistryNormalizesMissingAndContradictoryResults(t *testing.T) { require.NotEmpty(t, result.Error) } } + +type panickingTool struct{ BaseTool } + +func (t *panickingTool) Execute(context.Context, json.RawMessage) (*types.ToolResult, error) { + var items []int + _ = items[len(items)] // index out of range: a runtime panic like a buggy tool would raise + return nil, nil +} + +func TestRegistryRecoversToolPanic(t *testing.T) { + registry := NewToolRegistry() + registry.RegisterTool(&panickingTool{BaseTool: NewBaseTool("crashy", "", json.RawMessage(`{"type":"object"}`))}) + + result, err := registry.ExecuteTool(context.Background(), "crashy", json.RawMessage(`{}`)) + require.Error(t, err) + require.NotNil(t, result) + require.False(t, result.Success) + require.Equal(t, "tool crashy failed with an internal error", result.Error) + require.NotContains(t, result.Error, "index out of range") +}