fix(sandbox): treat in-use snapshot deletes as retryable conflicts

E2B refuses to delete a skill template while paused session sandboxes
still reference it, but reports that as HTTP 400. Classify it as
Conflict, leave the ledger row for the next sweep, and do not mark the
Langfuse span ERROR — this is expected until those sandboxes expire.
main
wizardchen 2026-08-31 19:33:41 +08:00 committed by lyingbug
parent 2dc8cdb334
commit af0aeb7cc2
12 changed files with 207 additions and 11 deletions

View File

@ -422,10 +422,10 @@ func configuredSandboxTTL(cfg *types.TenantSandboxConfig) time.Duration {
// image is never touched, nor is anything the ledger does not name: extras
// belong to other environments on a shared provider account.
//
// A live sandbox does not need the template it was created from in order to
// keep running, so the only reason to wait is in-flight creates that resolved
// the previous pointer. Twenty-four hours is far past every backend's default
// TTL; a config that sets a longer one extends the wait.
// Retention is a lower bound, not a guarantee the provider will accept the
// delete. Session sandboxes pause instead of dying when idle, and a paused
// sandbox still pins its template; a Conflict is left on the ledger for the
// next sweep.
func (s *TenantSkillService) PruneSupersededSnapshots(ctx context.Context) (int, error) {
if s == nil || s.skills == nil {
return 0, nil
@ -497,6 +497,12 @@ func (s *TenantSkillService) pruneConfigSnapshots(
continue
}
if err := deleter.DeleteSnapshot(ctx, row.SnapshotID); err != nil && !sandbox.IsRemoteNotFound(err) {
if sandbox.IsRemoteConflict(err) {
logger.Infof(ctx,
"[skill] snapshot %s still in use; leaving it until sandboxes release it: %s",
row.SnapshotID, sandbox.RemoteErrorDiagnostics(err))
continue
}
logger.Warnf(ctx, "[skill] delete superseded snapshot %s failed: %v", row.SnapshotID, err)
continue
}
@ -568,6 +574,12 @@ func (s *TenantSkillService) reapAbandonedBuilds(
continue
}
if err := deleter.DeleteSnapshot(ctx, found); err != nil && !sandbox.IsRemoteNotFound(err) {
if sandbox.IsRemoteConflict(err) {
logger.Infof(ctx,
"[skill] abandoned build %s of config %s still in use; leaving it until sandboxes release it: %s",
found, cfg.ID, sandbox.RemoteErrorDiagnostics(err))
continue
}
logger.Warnf(ctx, "[skill] delete abandoned build %s of config %s failed: %v",
found, cfg.ID, err)
continue

View File

@ -390,6 +390,28 @@ func TestPruneSupersededSnapshotsLeavesTheRowWhenDeleteFails(t *testing.T) {
"a failed provider delete must not be recorded as deleted")
}
func TestPruneSupersededSnapshotsRetriesWhenSnapshotStillInUse(t *testing.T) {
fx := newReaperFixture(t)
fx.svc.snapshotRetention = time.Hour
old := fx.now.Add(-2 * time.Hour)
fx.live("snap-live")
fx.superseded("sk-1", "snap-old", "", old)
fx.provider.deleteErr = sandbox.NewRemoteError(
sandbox.SandboxTypeE2B, "DeleteSnapshot", sandbox.RemoteErrorKindConflict,
"cannot delete template because there are paused sandboxes using it", nil,
)
n, err := fx.svc.PruneSupersededSnapshots(context.Background())
require.NoError(t, err)
require.Zero(t, n)
require.Empty(t, fx.provider.deleted)
rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")
require.NoError(t, err)
require.Equal(t, types.SkillSnapshotStateSuperseded, rows[0].State,
"an in-use template must stay on the ledger so the next sweep can retry")
}
func TestPruneSupersededSnapshotsTreatsMissingProviderSnapshotAsDeleted(t *testing.T) {
fx := newReaperFixture(t)
fx.svc.snapshotRetention = time.Hour

View File

@ -43,11 +43,10 @@ const (
skillInstallInFlightSkip = 3 * time.Minute
// skillSnapshotRetention is how long a superseded snapshot stays on the
// provider after the pointer has moved. Live sandboxes may still have
// been created from it (especially SkillRolloutNewSession); once they
// expire, the template is only a billed leftover. Twenty-four hours is
// well past every backend's default sandbox TTL. A config that sets a
// longer sandbox TTL extends this via snapshotRetentionFor.
// provider after the pointer has moved before prune even tries. Paused
// session sandboxes can pin the template past this window; prune retries
// on Conflict. A config that sets a longer sandbox TTL extends this via
// snapshotRetentionFor.
skillSnapshotRetention = 24 * time.Hour
// skillSnapshotTTLMargin is added on top of a config's own sandbox TTL

View File

@ -1123,6 +1123,7 @@ func normalizeCubeError(op string, err error) error {
kind = RemoteErrorKindUnavailable
}
}
kind = snapshotDeleteKind(op, kind, err.Error())
remoteErr := NewRemoteError(SandboxTypeCube, op, kind, err.Error(), err)
remoteErr.StatusCode = status
return remoteErr

View File

@ -470,6 +470,14 @@ func TestNormalizeCubeError(t *testing.T) {
{"bad gateway", "List", &cubesandbox.APIError{StatusCode: http.StatusBadGateway}, RemoteErrorKindUnavailable},
{"deadline", "Exec", context.DeadlineExceeded, RemoteErrorKindTimeout},
{"unknown", "List", errors.New("unknown"), RemoteErrorKindInternal},
{"delete snapshot in use", "DeleteSnapshot", &cubesandbox.APIError{
StatusCode: http.StatusBadRequest,
Message: "cannot delete template x because there are paused sandboxes using it",
}, RemoteErrorKindConflict},
{"delete snapshot bad id", "DeleteSnapshot", &cubesandbox.APIError{
StatusCode: http.StatusBadRequest,
Message: "invalid snapshot id",
}, RemoteErrorKindInvalidRequest},
}
for _, tt := range tests {

View File

@ -373,7 +373,7 @@ func dockerError(op string, err error) error {
return err
}
return &RemoteError{
Kind: dockerErrorKind(op, err),
Kind: snapshotDeleteKind(op, dockerErrorKind(op, err), err.Error()),
Provider: SandboxTypeDocker,
Op: op,
Message: err.Error(),

View File

@ -1253,6 +1253,7 @@ func normalizeE2BError(op string, err error) error {
}
}
}
kind = snapshotDeleteKind(op, kind, err.Error())
remoteErr := NewRemoteError(SandboxTypeE2B, op, kind, err.Error(), err)
remoteErr.StatusCode = status
return remoteErr

View File

@ -1391,6 +1391,14 @@ func TestNormalizeE2BError(t *testing.T) {
{"connect data loss", "List", connect.NewError(connect.CodeDataLoss, errors.New("data loss")), RemoteErrorKindInternal},
{"connect unknown", "List", connect.NewError(connect.CodeUnknown, errors.New("unknown")), RemoteErrorKindInternal},
{"unknown", "List", errors.New("mystery"), RemoteErrorKindInternal},
{"delete snapshot in use", "DeleteSnapshot", &e2b.Error{
StatusCode: http.StatusBadRequest,
Message: "cannot delete template 'tpl-1' because there are paused sandboxes using it",
}, RemoteErrorKindConflict},
{"delete snapshot bad id", "DeleteSnapshot", &e2b.Error{
StatusCode: http.StatusBadRequest,
Message: "invalid snapshot id",
}, RemoteErrorKindInvalidRequest},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

View File

@ -215,10 +215,24 @@ func (c *langfuseSnapshotClient) DeleteSnapshot(ctx context.Context, snapshotID
"snapshot_id": snapshotID,
}, nil)
err := inner.DeleteSnapshot(ctx, snapshotID)
span.Finish(nil, nil, err)
out, spanErr := snapshotDeleteSpanResult(err)
span.Finish(out, nil, spanErr)
return err
}
// snapshotDeleteSpanResult keeps an in-use Conflict off the ERROR status.
// Session sandboxes pause against the template they booted from, so the skill
// reaper hitting this every few minutes is expected retry, not a fault.
func snapshotDeleteSpanResult(err error) (map[string]interface{}, error) {
if !IsRemoteConflict(err) {
return nil, err
}
return map[string]interface{}{
"deferred": true,
"reason": "in_use",
}, nil
}
func (c *langfuseSnapshotClient) ListSnapshots(
ctx context.Context, sandboxID string,
) ([]RemoteSnapshotRef, error) {

View File

@ -61,3 +61,18 @@ func TestTruncateSandboxPreview(t *testing.T) {
require.True(t, strings.HasSuffix(got, "…"))
require.Equal(t, sandboxSpanPreviewRunes+1, utf8.RuneCountInString(got))
}
func TestSnapshotDeleteSpanResultDefersConflict(t *testing.T) {
out, spanErr := snapshotDeleteSpanResult(NewRemoteError(
SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict,
"paused sandboxes using it", nil,
))
require.NoError(t, spanErr, "in-use must not mark the Langfuse span ERROR")
require.Equal(t, true, out["deferred"])
require.Equal(t, "in_use", out["reason"])
boom := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInternal, "boom", nil)
out, spanErr = snapshotDeleteSpanResult(boom)
require.Equal(t, boom, spanErr)
require.Nil(t, out)
}

View File

@ -184,6 +184,37 @@ func IsRemoteInvalidRequest(err error) bool {
return remoteKind(err) == RemoteErrorKindInvalidRequest
}
// IsRemoteConflict reports whether the provider refused the call because the
// resource is busy (concurrent mutation, snapshot still referenced, …).
func IsRemoteConflict(err error) bool {
return remoteKind(err) == RemoteErrorKindConflict
}
// snapshotDeleteKind reclassifies a snapshot/template delete that failed
// because sandboxes still reference it. E2B returns that as HTTP 400
// invalid_request; it is Conflict: the caller did nothing wrong and should
// retry after those sandboxes (typically paused) go away.
func snapshotDeleteKind(op string, kind RemoteErrorKind, message string) RemoteErrorKind {
if op != "DeleteSnapshot" && op != "DeleteTemplate" {
return kind
}
if kind == RemoteErrorKindConflict || snapshotInUseBySandboxes(message) {
return RemoteErrorKindConflict
}
return kind
}
func snapshotInUseBySandboxes(message string) bool {
msg := strings.ToLower(message)
if strings.Contains(msg, "paused sandbox") {
return true
}
if strings.Contains(msg, "sandboxes using") {
return true
}
return strings.Contains(msg, "cannot delete template") && strings.Contains(msg, "using it")
}
// IsRemoteDirAlreadyExists reports whether MakeDir failed because the
// directory is already present. Cube's envd MakeDir is not mkdir -p: a
// directory created by a previous call (or by a shell `mkdir -p` in

View File

@ -177,3 +177,88 @@ func TestIsRemoteDirAlreadyExists(t *testing.T) {
})
}
}
func TestIsRemoteConflict(t *testing.T) {
t.Parallel()
if IsRemoteConflict(nil) {
t.Fatal("nil is not a conflict")
}
conflict := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict, "busy", nil)
if !IsRemoteConflict(conflict) {
t.Fatal("Conflict kind must match")
}
invalid := NewRemoteError(
SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInvalidRequest, "bad id", nil,
)
if IsRemoteConflict(invalid) {
t.Fatal("InvalidRequest must not match")
}
}
func TestSnapshotDeleteKindPromotesInUseToConflict(t *testing.T) {
t.Parallel()
inUse := "cannot delete template 'upfvuzpq0q6foo1mkpbkl' because there are paused sandboxes using it"
jsonBody := `e2b: status 400: {"code":400,"message":"` + inUse + `"}`
tests := []struct {
name string
op string
kind RemoteErrorKind
msg string
want RemoteErrorKind
}{
{
name: "e2b paused sandboxes 400",
op: "DeleteSnapshot",
kind: RemoteErrorKindInvalidRequest,
msg: inUse,
want: RemoteErrorKindConflict,
},
{
name: "json-wrapped e2b 400",
op: "DeleteSnapshot",
kind: RemoteErrorKindInvalidRequest,
msg: jsonBody,
want: RemoteErrorKindConflict,
},
{
name: "cube delete template",
op: "DeleteTemplate",
kind: RemoteErrorKindInvalidRequest,
msg: "cannot delete template x because there are sandboxes using it",
want: RemoteErrorKindConflict,
},
{
name: "already conflict stays conflict",
op: "DeleteSnapshot",
kind: RemoteErrorKindConflict,
msg: "image is in use",
want: RemoteErrorKindConflict,
},
{
name: "generic 400 stays invalid",
op: "DeleteSnapshot",
kind: RemoteErrorKindInvalidRequest,
msg: "invalid snapshot id",
want: RemoteErrorKindInvalidRequest,
},
{
name: "other ops are unchanged",
op: "Create",
kind: RemoteErrorKindInvalidRequest,
msg: inUse,
want: RemoteErrorKindInvalidRequest,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := snapshotDeleteKind(tt.op, tt.kind, tt.msg); got != tt.want {
t.Fatalf("snapshotDeleteKind() = %q, want %q", got, tt.want)
}
})
}
}