fix(sandbox): omit env overrides when creating Aliyun E2B sandboxes from snapshots (#4054)

* fix(sandbox): omit env overrides when creating Aliyun E2B sandboxes from snapshots

Aliyun Agent Sandbox rejects create-from-snapshot requests that resend env vars or function config. Drop those fields for Aliyun hosts, and retry once if another endpoint returns the same rejection.

* fix(sandbox): keep the snapshot rejection fixture within the line limit

---------

Co-authored-by: hailongzhao <hailongzhao@tencent.com>
main
nullptr-error 2026-10-09 14:24:54 +08:00 committed by GitHub
parent 65f88f4121
commit 52617f6410
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 220 additions and 16 deletions

View File

@ -39,6 +39,11 @@ type E2BRemoteClient struct {
templateID string
timeout time.Duration
// snapshotCreateImmutable is true for Aliyun Agent Sandbox. Creating a
// sandbox from a snapshot there rejects envVars and function-config
// overrides with HTTP 400, even when the values match the snapshot.
snapshotCreateImmutable bool
}
// NewE2BRemoteClient builds an E2B-backed RemoteSandboxClient from Config.
@ -127,15 +132,63 @@ func newE2BRemoteClient(
ttl = DefaultE2BSandboxTTL
}
return &E2BRemoteClient{
client: client,
inboundTokens: inboundTokens,
wsDialer: wsDialer,
desktopEnabled: cfg.DesktopEnabled,
templateID: strings.TrimSpace(cfg.E2BTemplate),
timeout: ttl,
client: client,
inboundTokens: inboundTokens,
wsDialer: wsDialer,
desktopEnabled: cfg.DesktopEnabled,
templateID: strings.TrimSpace(cfg.E2BTemplate),
timeout: ttl,
snapshotCreateImmutable: e2bSnapshotCreateImmutable(cfg.E2BAPIURL),
}, nil
}
// e2bSnapshotCreateImmutable reports Aliyun Agent Sandbox control planes.
// Their create-from-snapshot API refuses envVars and function-config fields
// that official E2B accepts.
func e2bSnapshotCreateImmutable(apiURL string) bool {
parsed, err := url.Parse(strings.TrimSpace(apiURL))
if err != nil {
return false
}
host := strings.ToLower(parsed.Hostname())
return host == "aliyuncs.com" || strings.HasSuffix(host, ".aliyuncs.com")
}
// stripSnapshotCreateOverrides removes the fields Aliyun rejects when the
// template is a snapshot. The snapshot already carries the env and function
// config of the sandbox it was taken from. The returned text names what was
// removed so the caller can log it; environment values are never included.
func stripSnapshotCreateOverrides(config *e2b.SandboxConfig) string {
var dropped []string
if len(config.EnvVars) > 0 {
names := make([]string, 0, len(config.EnvVars))
for name := range config.EnvVars {
names = append(names, name)
}
sort.Strings(names)
dropped = append(dropped, "envVars ["+strings.Join(names, ", ")+"]")
}
config.EnvVars = nil
if len(config.VolumeMounts) > 0 {
dropped = append(dropped, fmt.Sprintf("volumeMounts (%d)", len(config.VolumeMounts)))
}
config.VolumeMounts = nil
if config.AutoPauseMemory != nil {
dropped = append(dropped, "autoPauseMemory")
}
config.AutoPauseMemory = nil
return strings.Join(dropped, ", ")
}
func snapshotOverrideRejected(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "does not allow envvars") ||
strings.Contains(msg, "creating from a snapshot does not allow")
}
// e2bRemoteHandle is the RemoteSandboxHandle E2B returns. It carries the
// *e2b.Sandbox so subsequent envd calls can reuse its access token.
type e2bRemoteHandle struct {
@ -615,7 +668,28 @@ func (c *E2BRemoteClient) Create(
config.AutoPauseMemory = &autoPauseMemory
config.AutoResume = &e2b.AutoResumeConfig{Enabled: true}
}
// Aliyun stores env and function config inside the snapshot. Resending
// them is an override and the control plane answers 400 before the
// sandbox exists. Official E2B still accepts the fields, so they stay
// unless this client is pointed at Aliyun — or the provider says so.
if request.FromSnapshot && c.snapshotCreateImmutable {
if dropped := stripSnapshotCreateOverrides(&config); dropped != "" {
logger.Infof(ctx,
"e2b snapshot create omits fields Aliyun rejects: template %s: %s",
template, dropped,
)
}
}
sandbox, err := c.client.NewSandbox(ctx, config)
if err != nil && request.FromSnapshot && snapshotOverrideRejected(err) &&
(config.EnvVars != nil || len(config.VolumeMounts) > 0 || config.AutoPauseMemory != nil) {
dropped := stripSnapshotCreateOverrides(&config)
logger.Infof(ctx,
"e2b snapshot create rejected overrides; retrying without %s: template %s",
dropped, template,
)
sandbox, err = c.client.NewSandbox(ctx, config)
}
if err != nil {
return nil, normalizeE2BError("Create", err)
}

View File

@ -65,6 +65,9 @@ type e2bMockServer struct {
sandboxes map[string]map[string]any // sandboxID -> SandboxInfo JSON
snapshots map[string]map[string]any // snapshotID -> SnapshotInfo JSON
trafficAccessToken string
// rejectSnapshotEnv answers 400 when a create body includes envVars, matching
// Aliyun Agent Sandbox's snapshot restore rule.
rejectSnapshotEnv atomic.Bool
}
func newE2BMockServer(t *testing.T) *e2bMockServer {
@ -85,6 +88,9 @@ func (m *e2bMockServer) handle(w http.ResponseWriter, r *http.Request) {
case r.URL.Path == "/sandboxes" && r.Method == http.MethodPost:
m.createCount.Add(1)
body, _ := io.ReadAll(r.Body)
// Unmarshal merges into an existing map and would keep envVars from a
// rejected attempt, so each create starts from an empty body.
m.createBody = nil
_ = json.Unmarshal(body, &m.createBody)
id := "e2b-" + strconv.FormatInt(m.nextID.Add(1), 10)
m.sandboxes[id] = map[string]any{
@ -95,6 +101,17 @@ func (m *e2bMockServer) handle(w http.ResponseWriter, r *http.Request) {
"startedAt": time.Now().UTC().Format(time.RFC3339),
"metadata": m.createBody["metadata"],
}
if m.rejectSnapshotEnv.Load() {
if _, ok := m.createBody["envVars"]; ok {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(
`{"code":400,"message":"creating from a snapshot ` +
`does not allow envVars or function config overrides"}`,
))
return
}
}
m.mu.Lock()
trafficAccessToken := m.trafficAccessToken
m.mu.Unlock()
@ -1045,6 +1062,82 @@ func TestE2BRemoteClientCreateWritesMetadataAndPauseLifecycle(t *testing.T) {
}
func TestE2BSnapshotCreateDropsEnvVarsAfterProviderRejectsThem(t *testing.T) {
mock := newE2BMockServer(t)
mock.rejectSnapshotEnv.Store(true)
client := newTestE2BRemoteClient(t, mock)
handle, err := client.Create(context.Background(), RemoteCreateRequest{
TemplateID: "snap-1",
FromSnapshot: true,
EnvVars: map[string]string{"WEKNORA_SKILL_OUTPUT_DIR": "/workspace/output"},
Timeout: RemoteTimeoutPolicy{
Mode: RemoteTimeoutExplicit,
Value: time.Minute,
Action: RemoteOnTimeoutPause,
AutoResume: true,
},
})
require.NoError(t, err)
require.NotEmpty(t, handle.ID())
require.Equal(t, int32(2), mock.createCount.Load(), "the rejected create must be retried once without envVars")
_, hasEnv := mock.createBody["envVars"]
require.False(t, hasEnv, "the successful create must omit envVars: %#v", mock.createBody)
_, hasMemory := mock.createBody["autoPauseMemory"]
require.False(t, hasMemory, "function-config overrides must be omitted with envVars")
require.Equal(t, true, mock.createBody["autoPause"])
}
func TestE2BAliyunSnapshotCreateOmitsEnvVarsUpFront(t *testing.T) {
mock := newE2BMockServer(t)
mock.rejectSnapshotEnv.Store(true)
client := newTestE2BRemoteClient(t, mock)
client.snapshotCreateImmutable = true
_, err := client.Create(context.Background(), RemoteCreateRequest{
TemplateID: "snap-1",
FromSnapshot: true,
EnvVars: map[string]string{"WEKNORA_SKILL_OUTPUT_DIR": "/workspace/output", "HF_TOKEN": "secret"},
Timeout: RemoteTimeoutPolicy{
Mode: RemoteTimeoutExplicit,
Value: time.Minute,
Action: RemoteOnTimeoutPause,
AutoResume: true,
},
})
require.NoError(t, err)
require.Equal(t, int32(1), mock.createCount.Load())
_, hasEnv := mock.createBody["envVars"]
require.False(t, hasEnv)
_, hasMemory := mock.createBody["autoPauseMemory"]
require.False(t, hasMemory)
require.Equal(t, true, mock.createBody["autoResume"].(map[string]any)["enabled"])
}
func TestE2BSnapshotCreateKeepsEnvVarsOnOrdinaryProviders(t *testing.T) {
mock := newE2BMockServer(t)
client := newTestE2BRemoteClient(t, mock)
_, err := client.Create(context.Background(), RemoteCreateRequest{
TemplateID: "snap-1",
FromSnapshot: true,
EnvVars: map[string]string{"LANG": "C.UTF-8"},
})
require.NoError(t, err)
require.Equal(t, int32(1), mock.createCount.Load())
require.Equal(t, map[string]any{"LANG": "C.UTF-8"}, mock.createBody["envVars"])
}
func TestE2BSnapshotCreateImmutableHost(t *testing.T) {
require.True(t, e2bSnapshotCreateImmutable("https://api.cn-hangzhou.e2b.fc.aliyuncs.com"))
require.True(t, e2bSnapshotCreateImmutable("https://api.cn-beijing.sandbox.aliyuncs.com"))
require.False(t, e2bSnapshotCreateImmutable("https://api.e2b.app"))
require.False(t, e2bSnapshotCreateImmutable(""))
}
func TestE2BRemoteClientCreateValidatesTimeoutAction(t *testing.T) {
tests := []struct {
name string

View File

@ -114,6 +114,11 @@ type RemoteCreateRequest struct {
// TemplateID references the pre-baked sandbox template. Required.
TemplateID string
// FromSnapshot is set when TemplateID is a skill or fork snapshot rather
// than a base template. Aliyun Agent Sandbox rejects create-from-snapshot
// requests that resend env vars or function-config overrides.
FromSnapshot bool
// Timeout controls the idle-timeout policy.
Timeout RemoteTimeoutPolicy

View File

@ -422,6 +422,13 @@ type Config struct {
// E2BTemplate is the E2B template ID used at sandbox creation.
E2BTemplate string
// TemplateFromSnapshot reports that the boot target (E2BTemplate or
// CubeTemplate) was replaced by a skill snapshot. The env baked into that
// snapshot must not be sent again: Aliyun Agent Sandbox returns 400
// "creating from a snapshot does not allow envVars or function config
// overrides".
TemplateFromSnapshot bool
// E2BSandboxTTL is the E2B-side idle timeout hint.
E2BSandboxTTL time.Duration

View File

@ -52,6 +52,7 @@ func TestCreateAndBindUsesTemplateOverride(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "snap-1", client.lastCreateRequest.TemplateID)
require.True(t, client.lastCreateRequest.FromSnapshot)
}
func TestCreateAndBindKeepsConfigTemplateWithoutOverride(t *testing.T) {
@ -62,6 +63,7 @@ func TestCreateAndBindKeepsConfigTemplateWithoutOverride(t *testing.T) {
require.NoError(t, err)
require.Equal(t, lc.createRequest.TemplateID, client.lastCreateRequest.TemplateID)
require.False(t, client.lastCreateRequest.FromSnapshot)
}
// 最重要的一条:override 绝不能污染跨 session 复用的 l.createRequest。

View File

@ -448,6 +448,7 @@ func (l *remoteSessionLifecycle) createAndBind(
}
if strings.TrimSpace(override) != "" {
request.TemplateID = override
request.FromSnapshot = true
usedOverride = true
}
}

View File

@ -1571,9 +1571,10 @@ func buildSessionCreateRequest(provider RemoteProvider, cfg *Config) (RemoteCrea
ttl = DefaultCubeSandboxTTL
}
return RemoteCreateRequest{
TemplateID: cfg.CubeTemplate,
EnvVars: envVars,
Network: cfg.Network,
TemplateID: cfg.CubeTemplate,
FromSnapshot: cfg.TemplateFromSnapshot,
EnvVars: envVars,
Network: cfg.Network,
Timeout: RemoteTimeoutPolicy{
Mode: RemoteTimeoutExplicit,
Value: ttl,
@ -1588,9 +1589,10 @@ func buildSessionCreateRequest(provider RemoteProvider, cfg *Config) (RemoteCrea
ttl = DefaultE2BSandboxTTL
}
return RemoteCreateRequest{
TemplateID: cfg.E2BTemplate,
EnvVars: envVars,
Network: cfg.Network,
TemplateID: cfg.E2BTemplate,
FromSnapshot: cfg.TemplateFromSnapshot,
EnvVars: envVars,
Network: cfg.Network,
Timeout: RemoteTimeoutPolicy{
Mode: RemoteTimeoutExplicit,
Value: ttl,

View File

@ -505,6 +505,21 @@ func TestBuildSessionCreateRequestCarriesNetworkPolicy(t *testing.T) {
}
}
func TestBuildSessionCreateRequestMarksSkillSnapshot(t *testing.T) {
cfg := DefaultConfig()
cfg.Type = SandboxTypeE2B
cfg.E2BTemplate = "snap-1"
cfg.TemplateFromSnapshot = true
cfg.EnvVars = map[string]string{"HF_TOKEN": "secret"}
request, err := buildSessionCreateRequest(SandboxTypeE2B, cfg)
require.NoError(t, err)
require.True(t, request.FromSnapshot)
require.Equal(t, "secret", request.EnvVars["HF_TOKEN"])
require.NotEmpty(t, request.EnvVars["WEKNORA_SKILL_OUTPUT_DIR"])
}
func newSessionManagerExecTestHarness(t *testing.T) (*SessionBoundManager, *fakeRemoteClient) {
t.Helper()

View File

@ -135,22 +135,25 @@ func ResolveEffectiveConfig(
case SandboxTypeDocker:
applyDockerRuntimeDefaults(&effective)
}
// A skill snapshot is a template ID (Cube/E2B) or an image tag (Docker),
// so overriding that field here is the entire session-side change.
// Everything downstream keeps reading CubeTemplate / E2BTemplate /
// DockerImage and needs no knowledge of skills.
// A skill snapshot is a template ID (Cube/E2B) or an image tag (Docker).
// Downstream still reads CubeTemplate / E2BTemplate / DockerImage.
// TemplateFromSnapshot tells the E2B adapter this ID is a snapshot, so it
// can omit the env and function-config fields Aliyun refuses to override.
effective.TemplateFromSnapshot = false
switch effective.Type {
case SandboxTypeCube:
if snapshot := skillImageTemplateOverride(
tenantCfg.SkillImage, "cube", effective.CubeAPIKey, effective.CubeAPIURL,
); snapshot != "" {
effective.CubeTemplate = snapshot
effective.TemplateFromSnapshot = true
}
case SandboxTypeE2B:
if snapshot := skillImageTemplateOverride(
tenantCfg.SkillImage, "e2b", effective.E2BAPIKey, effective.E2BAPIURL,
); snapshot != "" {
effective.E2BTemplate = snapshot
effective.TemplateFromSnapshot = true
}
case SandboxTypeDocker:
// Deliberately computed from the STORED docker block, not from

View File

@ -672,6 +672,7 @@ func TestResolveEffectiveConfigUsesSkillSnapshotAsE2BTemplate(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "snap-1", eff.E2BTemplate)
require.True(t, eff.TemplateFromSnapshot)
})
t.Run("fingerprint mismatch falls back to the base template", func(t *testing.T) {
@ -685,6 +686,7 @@ func TestResolveEffectiveConfigUsesSkillSnapshotAsE2BTemplate(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "tpl-base", eff.E2BTemplate,
"a snapshot whose fingerprint was computed for cube must not override e2b; the session must still boot")
require.False(t, eff.TemplateFromSnapshot)
})
t.Run("empty snapshot keeps the base template", func(t *testing.T) {