feat(sandbox): enhance logging for sandbox operations and error diagnostics
- Added detailed logging for sandbox creation and execution failures in `sandbox_check.go` and `cube_remote_client.go`. - Introduced `logCubeSandboxCreated` and `logCubeDataPlaneExec` functions to capture relevant context during sandbox operations. - Improved `RemoteErrorDiagnostics` to format error messages with additional context, including operation type and HTTP status. - Updated tests to validate new error diagnostics functionality.main
parent
d6d177f3ab
commit
4c397ffc4d
|
|
@ -407,6 +407,10 @@ func (h *SystemHandler) runDeepSandboxCheck(
|
|||
}
|
||||
}()
|
||||
result.add("template_exists", true, "", 0)
|
||||
logger.Infof(ctx,
|
||||
"[SandboxCheck] probe sandbox created id=%s provider=%s template=%s",
|
||||
handle.ID(), result.Provider, sandbox.EffectiveTemplateID(cfg),
|
||||
)
|
||||
|
||||
const marker = "weknora-ok"
|
||||
start := time.Now()
|
||||
|
|
@ -419,6 +423,10 @@ func (h *SystemHandler) runDeepSandboxCheck(
|
|||
latency := time.Since(start).Milliseconds()
|
||||
switch {
|
||||
case err != nil:
|
||||
logger.Warnf(ctx,
|
||||
"[SandboxCheck] sandbox_exec failed sandbox=%s latency_ms=%d ui_reason=%s detail=%s",
|
||||
handle.ID(), latency, sandboxCheckReason(err), sandbox.RemoteErrorDiagnostics(err),
|
||||
)
|
||||
result.add("sandbox_exec", false, sandboxCheckReason(err), latency)
|
||||
result.skip("egress_available", skipReasonSandboxExecFailed)
|
||||
return
|
||||
|
|
@ -427,6 +435,11 @@ func (h *SystemHandler) runDeepSandboxCheck(
|
|||
result.skip("egress_available", skipReasonSandboxExecFailed)
|
||||
return
|
||||
case !strings.Contains(execResult.Stdout, marker):
|
||||
logger.Warnf(ctx,
|
||||
"[SandboxCheck] sandbox_exec probe mismatch sandbox=%s exit=%d killed=%v stdout=%q stderr=%q",
|
||||
handle.ID(), execResult.ExitCode, execResult.Killed,
|
||||
firstProbeLine(execResult.Stdout), firstProbeLine(execResult.Stderr),
|
||||
)
|
||||
result.add("sandbox_exec", false, describeProbeMismatch(
|
||||
execResult.ExitCode, execResult.Killed,
|
||||
execResult.Stdout, execResult.Stderr, "",
|
||||
|
|
|
|||
|
|
@ -338,6 +338,7 @@ func (c *CubeRemoteClient) Create(
|
|||
if sb == nil || sb.SandboxID == "" {
|
||||
return nil, errors.New("cube api: create sandbox: empty sandboxID")
|
||||
}
|
||||
logCubeSandboxCreated(ctx, c, sb, network.AllowPublicTraffic)
|
||||
return &cubeRemoteHandle{
|
||||
sb: sb,
|
||||
metadata: cloneMetadata(request.Metadata),
|
||||
|
|
@ -468,6 +469,8 @@ func (c *CubeRemoteClient) Exec(
|
|||
envs = map[string]string{}
|
||||
}
|
||||
|
||||
logCubeDataPlaneExec(ctx, c, sb, request.User, line)
|
||||
|
||||
startedAt := time.Now()
|
||||
// User comes from the neutral request rather than being hardcoded: running
|
||||
// everything as root silently defeats file-mode protections on shared
|
||||
|
|
@ -489,7 +492,12 @@ func (c *CubeRemoteClient) Exec(
|
|||
ExitCode: -1,
|
||||
}, nil
|
||||
}
|
||||
return nil, normalizeCubeError("Exec", execErr)
|
||||
normalized := normalizeCubeError("Exec", execErr)
|
||||
logger.Warnf(ctx,
|
||||
"[CubeRemote] data-plane exec failed sandbox=%s detail=%s",
|
||||
sb.SandboxID, RemoteErrorDiagnostics(normalized),
|
||||
)
|
||||
return nil, normalized
|
||||
}
|
||||
if sdkResult == nil {
|
||||
return nil, NewRemoteError(
|
||||
|
|
@ -864,6 +872,7 @@ func normalizeCubeError(op string, err error) error {
|
|||
}
|
||||
|
||||
kind := RemoteErrorKindInternal
|
||||
status := 0
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
kind = RemoteErrorKindTimeout
|
||||
|
|
@ -882,13 +891,87 @@ func normalizeCubeError(op string, err error) error {
|
|||
kind = RemoteErrorKindNotFound
|
||||
case errors.As(err, &apiErr):
|
||||
kind = httpErrorKind(op, apiErr.StatusCode)
|
||||
status = apiErr.StatusCode
|
||||
case errors.As(err, &netErr) && netErr.Timeout():
|
||||
kind = RemoteErrorKindTimeout
|
||||
case errors.As(err, &netErr):
|
||||
kind = RemoteErrorKindUnavailable
|
||||
}
|
||||
}
|
||||
return NewRemoteError(SandboxTypeCube, op, kind, err.Error(), err)
|
||||
remoteErr := NewRemoteError(SandboxTypeCube, op, kind, err.Error(), err)
|
||||
remoteErr.StatusCode = status
|
||||
return remoteErr
|
||||
}
|
||||
|
||||
// logCubeSandboxCreated records create-time data-plane fields operators need
|
||||
// when exec fails with a generic auth error. Token values are never logged.
|
||||
func logCubeSandboxCreated(
|
||||
ctx context.Context,
|
||||
client *CubeRemoteClient,
|
||||
sb *cubesandbox.Sandbox,
|
||||
allowPublicTraffic *bool,
|
||||
) {
|
||||
if sb == nil || client == nil {
|
||||
return
|
||||
}
|
||||
publicTraffic := "server_default"
|
||||
if allowPublicTraffic != nil {
|
||||
publicTraffic = fmt.Sprintf("%t", *allowPublicTraffic)
|
||||
}
|
||||
logger.Infof(ctx,
|
||||
"[CubeRemote] sandbox created id=%s template=%s domain=%s envd_host=%s "+
|
||||
"api_url=%s proxy_url=%s api_key=%s envd_token=%s traffic_token=%s "+
|
||||
"allow_public_traffic=%s",
|
||||
sb.SandboxID,
|
||||
sb.TemplateID,
|
||||
cubeSandboxDomain(client, sb),
|
||||
sb.GetHost(CubeEnvdPort),
|
||||
client.config.CubeAPIURL,
|
||||
client.config.CubeProxyURL,
|
||||
cubeCredentialPresence(client.config.CubeAPIKey),
|
||||
cubeCredentialPresence(sb.EnvdAccessToken),
|
||||
cubeCredentialPresence(sb.TrafficAccessToken),
|
||||
publicTraffic,
|
||||
)
|
||||
}
|
||||
|
||||
func logCubeDataPlaneExec(
|
||||
ctx context.Context,
|
||||
client *CubeRemoteClient,
|
||||
sb *cubesandbox.Sandbox,
|
||||
execUser, commandLine string,
|
||||
) {
|
||||
if sb == nil || client == nil {
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx,
|
||||
"[CubeRemote] data-plane exec sandbox=%s envd_host=%s exec_user=%s "+
|
||||
"api_key=%s envd_token=%s traffic_token=%s cmd=%q",
|
||||
sb.SandboxID,
|
||||
sb.GetHost(CubeEnvdPort),
|
||||
strings.TrimSpace(execUser),
|
||||
cubeCredentialPresence(client.config.CubeAPIKey),
|
||||
cubeCredentialPresence(sb.EnvdAccessToken),
|
||||
cubeCredentialPresence(sb.TrafficAccessToken),
|
||||
commandLine,
|
||||
)
|
||||
}
|
||||
|
||||
func cubeSandboxDomain(client *CubeRemoteClient, sb *cubesandbox.Sandbox) string {
|
||||
if sb != nil && strings.TrimSpace(sb.Domain) != "" {
|
||||
return sb.Domain
|
||||
}
|
||||
if client != nil && client.sandboxDomain != "" {
|
||||
return client.sandboxDomain
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func cubeCredentialPresence(value string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return "absent"
|
||||
}
|
||||
return "present"
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// RemoteErrorKind is the stable, provider-neutral classification of a
|
||||
|
|
@ -130,6 +131,29 @@ func NewRemoteError(provider RemoteProvider, op string, kind RemoteErrorKind, me
|
|||
}
|
||||
}
|
||||
|
||||
// RemoteErrorDiagnostics formats err for logs. When err wraps a RemoteError
|
||||
// it includes kind, op, HTTP status, and message without re-parsing text.
|
||||
func RemoteErrorDiagnostics(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
var re *RemoteError
|
||||
if !errors.As(err, &re) {
|
||||
return err.Error()
|
||||
}
|
||||
parts := []string{string(re.Kind)}
|
||||
if re.Op != "" {
|
||||
parts = append(parts, "op="+re.Op)
|
||||
}
|
||||
if re.StatusCode != 0 {
|
||||
parts = append(parts, fmt.Sprintf("http=%d", re.StatusCode))
|
||||
}
|
||||
if re.Message != "" {
|
||||
parts = append(parts, re.Message)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// remoteKind extracts the Kind from err, or "" when err is not a
|
||||
// *RemoteError. Wraps errors.As so callers can pass any error value.
|
||||
func remoteKind(err error) RemoteErrorKind {
|
||||
|
|
|
|||
|
|
@ -111,3 +111,20 @@ func TestRemoteErrorRetainsCause(t *testing.T) {
|
|||
t.Fatalf("Error() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteErrorDiagnostics(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
remote := NewRemoteError(
|
||||
SandboxTypeCube, "Exec", RemoteErrorKindAuthentication,
|
||||
"HTTP 403", errors.New("forbidden"),
|
||||
)
|
||||
remote.StatusCode = 403
|
||||
got := RemoteErrorDiagnostics(remote)
|
||||
if got != "authentication op=Exec http=403 HTTP 403" {
|
||||
t.Fatalf("RemoteErrorDiagnostics() = %q", got)
|
||||
}
|
||||
if RemoteErrorDiagnostics(errors.New("plain")) != "plain" {
|
||||
t.Fatal("expected plain error text")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ const (
|
|||
// DefaultDockerImage therefore always fails the probe with "connection
|
||||
// refused" — E2B gets away with that image because its own builder injects
|
||||
// envd, and the Docker backend never needs one.
|
||||
DefaultCubeTemplateImage = "wechatopenai/weknora-sandbox:latest-cube"
|
||||
DefaultCubeTemplateImage = "wechatopenai/weknora-sandbox:main-cube"
|
||||
|
||||
// CubeEnvdPort is the port envd listens on inside a Cube sandbox. It carries
|
||||
// the readiness probe as well as every exec and filesystem call, and the
|
||||
|
|
|
|||
Loading…
Reference in New Issue