From af0aeb7cc284d76e2c59b4eaf686132e0d5b8caa Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 31 Aug 2026 19:33:41 +0800 Subject: [PATCH] fix(sandbox): treat in-use snapshot deletes as retryable conflicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E2B refuses to delete a skill template while paused session sandboxes still reference it, but reports that as HTTP 400. Classify it as Conflict, leave the ledger row for the next sweep, and do not mark the Langfuse span ERROR — this is expected until those sandboxes expire. --- .../service/tenant_skill_reaper.go | 20 ++++- .../service/tenant_skill_reaper_test.go | 22 +++++ .../service/tenant_skill_service.go | 9 +- internal/sandbox/cube_remote_client.go | 1 + internal/sandbox/cube_remote_client_test.go | 8 ++ internal/sandbox/docker_engine.go | 2 +- internal/sandbox/e2b_remote_client.go | 1 + internal/sandbox/e2b_remote_client_test.go | 8 ++ internal/sandbox/langfuse.go | 16 +++- internal/sandbox/langfuse_test.go | 15 ++++ internal/sandbox/remote_errors.go | 31 +++++++ internal/sandbox/remote_errors_test.go | 85 +++++++++++++++++++ 12 files changed, 207 insertions(+), 11 deletions(-) diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.go index e6f70f69..fada45db 100644 --- a/internal/application/service/tenant_skill_reaper.go +++ b/internal/application/service/tenant_skill_reaper.go @@ -422,10 +422,10 @@ func configuredSandboxTTL(cfg *types.TenantSandboxConfig) time.Duration { // image is never touched, nor is anything the ledger does not name: extras // belong to other environments on a shared provider account. // -// A live sandbox does not need the template it was created from in order to -// keep running, so the only reason to wait is in-flight creates that resolved -// the previous pointer. Twenty-four hours is far past every backend's default -// TTL; a config that sets a longer one extends the wait. +// Retention is a lower bound, not a guarantee the provider will accept the +// delete. Session sandboxes pause instead of dying when idle, and a paused +// sandbox still pins its template; a Conflict is left on the ledger for the +// next sweep. func (s *TenantSkillService) PruneSupersededSnapshots(ctx context.Context) (int, error) { if s == nil || s.skills == nil { return 0, nil @@ -497,6 +497,12 @@ func (s *TenantSkillService) pruneConfigSnapshots( continue } if err := deleter.DeleteSnapshot(ctx, row.SnapshotID); err != nil && !sandbox.IsRemoteNotFound(err) { + if sandbox.IsRemoteConflict(err) { + logger.Infof(ctx, + "[skill] snapshot %s still in use; leaving it until sandboxes release it: %s", + row.SnapshotID, sandbox.RemoteErrorDiagnostics(err)) + continue + } logger.Warnf(ctx, "[skill] delete superseded snapshot %s failed: %v", row.SnapshotID, err) continue } @@ -568,6 +574,12 @@ func (s *TenantSkillService) reapAbandonedBuilds( continue } if err := deleter.DeleteSnapshot(ctx, found); err != nil && !sandbox.IsRemoteNotFound(err) { + if sandbox.IsRemoteConflict(err) { + logger.Infof(ctx, + "[skill] abandoned build %s of config %s still in use; leaving it until sandboxes release it: %s", + found, cfg.ID, sandbox.RemoteErrorDiagnostics(err)) + continue + } logger.Warnf(ctx, "[skill] delete abandoned build %s of config %s failed: %v", found, cfg.ID, err) continue diff --git a/internal/application/service/tenant_skill_reaper_test.go b/internal/application/service/tenant_skill_reaper_test.go index f6965751..c56f34ee 100644 --- a/internal/application/service/tenant_skill_reaper_test.go +++ b/internal/application/service/tenant_skill_reaper_test.go @@ -390,6 +390,28 @@ func TestPruneSupersededSnapshotsLeavesTheRowWhenDeleteFails(t *testing.T) { "a failed provider delete must not be recorded as deleted") } +func TestPruneSupersededSnapshotsRetriesWhenSnapshotStillInUse(t *testing.T) { + fx := newReaperFixture(t) + fx.svc.snapshotRetention = time.Hour + old := fx.now.Add(-2 * time.Hour) + fx.live("snap-live") + fx.superseded("sk-1", "snap-old", "", old) + fx.provider.deleteErr = sandbox.NewRemoteError( + sandbox.SandboxTypeE2B, "DeleteSnapshot", sandbox.RemoteErrorKindConflict, + "cannot delete template because there are paused sandboxes using it", nil, + ) + + n, err := fx.svc.PruneSupersededSnapshots(context.Background()) + + require.NoError(t, err) + require.Zero(t, n) + require.Empty(t, fx.provider.deleted) + rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1") + require.NoError(t, err) + require.Equal(t, types.SkillSnapshotStateSuperseded, rows[0].State, + "an in-use template must stay on the ledger so the next sweep can retry") +} + func TestPruneSupersededSnapshotsTreatsMissingProviderSnapshotAsDeleted(t *testing.T) { fx := newReaperFixture(t) fx.svc.snapshotRetention = time.Hour diff --git a/internal/application/service/tenant_skill_service.go b/internal/application/service/tenant_skill_service.go index 247775cc..6dabe2e6 100644 --- a/internal/application/service/tenant_skill_service.go +++ b/internal/application/service/tenant_skill_service.go @@ -43,11 +43,10 @@ const ( skillInstallInFlightSkip = 3 * time.Minute // skillSnapshotRetention is how long a superseded snapshot stays on the - // provider after the pointer has moved. Live sandboxes may still have - // been created from it (especially SkillRolloutNewSession); once they - // expire, the template is only a billed leftover. Twenty-four hours is - // well past every backend's default sandbox TTL. A config that sets a - // longer sandbox TTL extends this via snapshotRetentionFor. + // provider after the pointer has moved before prune even tries. Paused + // session sandboxes can pin the template past this window; prune retries + // on Conflict. A config that sets a longer sandbox TTL extends this via + // snapshotRetentionFor. skillSnapshotRetention = 24 * time.Hour // skillSnapshotTTLMargin is added on top of a config's own sandbox TTL diff --git a/internal/sandbox/cube_remote_client.go b/internal/sandbox/cube_remote_client.go index c10de2fb..562ae6c2 100644 --- a/internal/sandbox/cube_remote_client.go +++ b/internal/sandbox/cube_remote_client.go @@ -1123,6 +1123,7 @@ func normalizeCubeError(op string, err error) error { kind = RemoteErrorKindUnavailable } } + kind = snapshotDeleteKind(op, kind, err.Error()) remoteErr := NewRemoteError(SandboxTypeCube, op, kind, err.Error(), err) remoteErr.StatusCode = status return remoteErr diff --git a/internal/sandbox/cube_remote_client_test.go b/internal/sandbox/cube_remote_client_test.go index c5eca910..0e487bcd 100644 --- a/internal/sandbox/cube_remote_client_test.go +++ b/internal/sandbox/cube_remote_client_test.go @@ -470,6 +470,14 @@ func TestNormalizeCubeError(t *testing.T) { {"bad gateway", "List", &cubesandbox.APIError{StatusCode: http.StatusBadGateway}, RemoteErrorKindUnavailable}, {"deadline", "Exec", context.DeadlineExceeded, RemoteErrorKindTimeout}, {"unknown", "List", errors.New("unknown"), RemoteErrorKindInternal}, + {"delete snapshot in use", "DeleteSnapshot", &cubesandbox.APIError{ + StatusCode: http.StatusBadRequest, + Message: "cannot delete template x because there are paused sandboxes using it", + }, RemoteErrorKindConflict}, + {"delete snapshot bad id", "DeleteSnapshot", &cubesandbox.APIError{ + StatusCode: http.StatusBadRequest, + Message: "invalid snapshot id", + }, RemoteErrorKindInvalidRequest}, } for _, tt := range tests { diff --git a/internal/sandbox/docker_engine.go b/internal/sandbox/docker_engine.go index c5e07d13..0e08558b 100644 --- a/internal/sandbox/docker_engine.go +++ b/internal/sandbox/docker_engine.go @@ -373,7 +373,7 @@ func dockerError(op string, err error) error { return err } return &RemoteError{ - Kind: dockerErrorKind(op, err), + Kind: snapshotDeleteKind(op, dockerErrorKind(op, err), err.Error()), Provider: SandboxTypeDocker, Op: op, Message: err.Error(), diff --git a/internal/sandbox/e2b_remote_client.go b/internal/sandbox/e2b_remote_client.go index f835044f..1d93906c 100644 --- a/internal/sandbox/e2b_remote_client.go +++ b/internal/sandbox/e2b_remote_client.go @@ -1253,6 +1253,7 @@ func normalizeE2BError(op string, err error) error { } } } + kind = snapshotDeleteKind(op, kind, err.Error()) remoteErr := NewRemoteError(SandboxTypeE2B, op, kind, err.Error(), err) remoteErr.StatusCode = status return remoteErr diff --git a/internal/sandbox/e2b_remote_client_test.go b/internal/sandbox/e2b_remote_client_test.go index 09737b58..e2b43b88 100644 --- a/internal/sandbox/e2b_remote_client_test.go +++ b/internal/sandbox/e2b_remote_client_test.go @@ -1391,6 +1391,14 @@ func TestNormalizeE2BError(t *testing.T) { {"connect data loss", "List", connect.NewError(connect.CodeDataLoss, errors.New("data loss")), RemoteErrorKindInternal}, {"connect unknown", "List", connect.NewError(connect.CodeUnknown, errors.New("unknown")), RemoteErrorKindInternal}, {"unknown", "List", errors.New("mystery"), RemoteErrorKindInternal}, + {"delete snapshot in use", "DeleteSnapshot", &e2b.Error{ + StatusCode: http.StatusBadRequest, + Message: "cannot delete template 'tpl-1' because there are paused sandboxes using it", + }, RemoteErrorKindConflict}, + {"delete snapshot bad id", "DeleteSnapshot", &e2b.Error{ + StatusCode: http.StatusBadRequest, + Message: "invalid snapshot id", + }, RemoteErrorKindInvalidRequest}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/sandbox/langfuse.go b/internal/sandbox/langfuse.go index 8cab1e67..363b653b 100644 --- a/internal/sandbox/langfuse.go +++ b/internal/sandbox/langfuse.go @@ -215,10 +215,24 @@ func (c *langfuseSnapshotClient) DeleteSnapshot(ctx context.Context, snapshotID "snapshot_id": snapshotID, }, nil) err := inner.DeleteSnapshot(ctx, snapshotID) - span.Finish(nil, nil, err) + out, spanErr := snapshotDeleteSpanResult(err) + span.Finish(out, nil, spanErr) return err } +// snapshotDeleteSpanResult keeps an in-use Conflict off the ERROR status. +// Session sandboxes pause against the template they booted from, so the skill +// reaper hitting this every few minutes is expected retry, not a fault. +func snapshotDeleteSpanResult(err error) (map[string]interface{}, error) { + if !IsRemoteConflict(err) { + return nil, err + } + return map[string]interface{}{ + "deferred": true, + "reason": "in_use", + }, nil +} + func (c *langfuseSnapshotClient) ListSnapshots( ctx context.Context, sandboxID string, ) ([]RemoteSnapshotRef, error) { diff --git a/internal/sandbox/langfuse_test.go b/internal/sandbox/langfuse_test.go index 84c11d27..8687bbf1 100644 --- a/internal/sandbox/langfuse_test.go +++ b/internal/sandbox/langfuse_test.go @@ -61,3 +61,18 @@ func TestTruncateSandboxPreview(t *testing.T) { require.True(t, strings.HasSuffix(got, "…")) require.Equal(t, sandboxSpanPreviewRunes+1, utf8.RuneCountInString(got)) } + +func TestSnapshotDeleteSpanResultDefersConflict(t *testing.T) { + out, spanErr := snapshotDeleteSpanResult(NewRemoteError( + SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict, + "paused sandboxes using it", nil, + )) + require.NoError(t, spanErr, "in-use must not mark the Langfuse span ERROR") + require.Equal(t, true, out["deferred"]) + require.Equal(t, "in_use", out["reason"]) + + boom := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInternal, "boom", nil) + out, spanErr = snapshotDeleteSpanResult(boom) + require.Equal(t, boom, spanErr) + require.Nil(t, out) +} diff --git a/internal/sandbox/remote_errors.go b/internal/sandbox/remote_errors.go index b5041240..b89eec0d 100644 --- a/internal/sandbox/remote_errors.go +++ b/internal/sandbox/remote_errors.go @@ -184,6 +184,37 @@ func IsRemoteInvalidRequest(err error) bool { return remoteKind(err) == RemoteErrorKindInvalidRequest } +// IsRemoteConflict reports whether the provider refused the call because the +// resource is busy (concurrent mutation, snapshot still referenced, …). +func IsRemoteConflict(err error) bool { + return remoteKind(err) == RemoteErrorKindConflict +} + +// snapshotDeleteKind reclassifies a snapshot/template delete that failed +// because sandboxes still reference it. E2B returns that as HTTP 400 +// invalid_request; it is Conflict: the caller did nothing wrong and should +// retry after those sandboxes (typically paused) go away. +func snapshotDeleteKind(op string, kind RemoteErrorKind, message string) RemoteErrorKind { + if op != "DeleteSnapshot" && op != "DeleteTemplate" { + return kind + } + if kind == RemoteErrorKindConflict || snapshotInUseBySandboxes(message) { + return RemoteErrorKindConflict + } + return kind +} + +func snapshotInUseBySandboxes(message string) bool { + msg := strings.ToLower(message) + if strings.Contains(msg, "paused sandbox") { + return true + } + if strings.Contains(msg, "sandboxes using") { + return true + } + return strings.Contains(msg, "cannot delete template") && strings.Contains(msg, "using it") +} + // IsRemoteDirAlreadyExists reports whether MakeDir failed because the // directory is already present. Cube's envd MakeDir is not mkdir -p: a // directory created by a previous call (or by a shell `mkdir -p` in diff --git a/internal/sandbox/remote_errors_test.go b/internal/sandbox/remote_errors_test.go index cc7df831..0ad3dee9 100644 --- a/internal/sandbox/remote_errors_test.go +++ b/internal/sandbox/remote_errors_test.go @@ -177,3 +177,88 @@ func TestIsRemoteDirAlreadyExists(t *testing.T) { }) } } + +func TestIsRemoteConflict(t *testing.T) { + t.Parallel() + + if IsRemoteConflict(nil) { + t.Fatal("nil is not a conflict") + } + conflict := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict, "busy", nil) + if !IsRemoteConflict(conflict) { + t.Fatal("Conflict kind must match") + } + invalid := NewRemoteError( + SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInvalidRequest, "bad id", nil, + ) + if IsRemoteConflict(invalid) { + t.Fatal("InvalidRequest must not match") + } +} + +func TestSnapshotDeleteKindPromotesInUseToConflict(t *testing.T) { + t.Parallel() + + inUse := "cannot delete template 'upfvuzpq0q6foo1mkpbkl' because there are paused sandboxes using it" + jsonBody := `e2b: status 400: {"code":400,"message":"` + inUse + `"}` + + tests := []struct { + name string + op string + kind RemoteErrorKind + msg string + want RemoteErrorKind + }{ + { + name: "e2b paused sandboxes 400", + op: "DeleteSnapshot", + kind: RemoteErrorKindInvalidRequest, + msg: inUse, + want: RemoteErrorKindConflict, + }, + { + name: "json-wrapped e2b 400", + op: "DeleteSnapshot", + kind: RemoteErrorKindInvalidRequest, + msg: jsonBody, + want: RemoteErrorKindConflict, + }, + { + name: "cube delete template", + op: "DeleteTemplate", + kind: RemoteErrorKindInvalidRequest, + msg: "cannot delete template x because there are sandboxes using it", + want: RemoteErrorKindConflict, + }, + { + name: "already conflict stays conflict", + op: "DeleteSnapshot", + kind: RemoteErrorKindConflict, + msg: "image is in use", + want: RemoteErrorKindConflict, + }, + { + name: "generic 400 stays invalid", + op: "DeleteSnapshot", + kind: RemoteErrorKindInvalidRequest, + msg: "invalid snapshot id", + want: RemoteErrorKindInvalidRequest, + }, + { + name: "other ops are unchanged", + op: "Create", + kind: RemoteErrorKindInvalidRequest, + msg: inUse, + want: RemoteErrorKindInvalidRequest, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := snapshotDeleteKind(tt.op, tt.kind, tt.msg); got != tt.want { + t.Fatalf("snapshotDeleteKind() = %q, want %q", got, tt.want) + } + }) + } +}