fix(sandbox): treat in-use snapshot deletes as retryable conflicts
E2B refuses to delete a skill template while paused session sandboxes still reference it, but reports that as HTTP 400. Classify it as Conflict, leave the ledger row for the next sweep, and do not mark the Langfuse span ERROR — this is expected until those sandboxes expire.main
parent
2dc8cdb334
commit
af0aeb7cc2
|
|
@ -422,10 +422,10 @@ func configuredSandboxTTL(cfg *types.TenantSandboxConfig) time.Duration {
|
|||
// image is never touched, nor is anything the ledger does not name: extras
|
||||
// belong to other environments on a shared provider account.
|
||||
//
|
||||
// A live sandbox does not need the template it was created from in order to
|
||||
// keep running, so the only reason to wait is in-flight creates that resolved
|
||||
// the previous pointer. Twenty-four hours is far past every backend's default
|
||||
// TTL; a config that sets a longer one extends the wait.
|
||||
// Retention is a lower bound, not a guarantee the provider will accept the
|
||||
// delete. Session sandboxes pause instead of dying when idle, and a paused
|
||||
// sandbox still pins its template; a Conflict is left on the ledger for the
|
||||
// next sweep.
|
||||
func (s *TenantSkillService) PruneSupersededSnapshots(ctx context.Context) (int, error) {
|
||||
if s == nil || s.skills == nil {
|
||||
return 0, nil
|
||||
|
|
@ -497,6 +497,12 @@ func (s *TenantSkillService) pruneConfigSnapshots(
|
|||
continue
|
||||
}
|
||||
if err := deleter.DeleteSnapshot(ctx, row.SnapshotID); err != nil && !sandbox.IsRemoteNotFound(err) {
|
||||
if sandbox.IsRemoteConflict(err) {
|
||||
logger.Infof(ctx,
|
||||
"[skill] snapshot %s still in use; leaving it until sandboxes release it: %s",
|
||||
row.SnapshotID, sandbox.RemoteErrorDiagnostics(err))
|
||||
continue
|
||||
}
|
||||
logger.Warnf(ctx, "[skill] delete superseded snapshot %s failed: %v", row.SnapshotID, err)
|
||||
continue
|
||||
}
|
||||
|
|
@ -568,6 +574,12 @@ func (s *TenantSkillService) reapAbandonedBuilds(
|
|||
continue
|
||||
}
|
||||
if err := deleter.DeleteSnapshot(ctx, found); err != nil && !sandbox.IsRemoteNotFound(err) {
|
||||
if sandbox.IsRemoteConflict(err) {
|
||||
logger.Infof(ctx,
|
||||
"[skill] abandoned build %s of config %s still in use; leaving it until sandboxes release it: %s",
|
||||
found, cfg.ID, sandbox.RemoteErrorDiagnostics(err))
|
||||
continue
|
||||
}
|
||||
logger.Warnf(ctx, "[skill] delete abandoned build %s of config %s failed: %v",
|
||||
found, cfg.ID, err)
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -390,6 +390,28 @@ func TestPruneSupersededSnapshotsLeavesTheRowWhenDeleteFails(t *testing.T) {
|
|||
"a failed provider delete must not be recorded as deleted")
|
||||
}
|
||||
|
||||
func TestPruneSupersededSnapshotsRetriesWhenSnapshotStillInUse(t *testing.T) {
|
||||
fx := newReaperFixture(t)
|
||||
fx.svc.snapshotRetention = time.Hour
|
||||
old := fx.now.Add(-2 * time.Hour)
|
||||
fx.live("snap-live")
|
||||
fx.superseded("sk-1", "snap-old", "", old)
|
||||
fx.provider.deleteErr = sandbox.NewRemoteError(
|
||||
sandbox.SandboxTypeE2B, "DeleteSnapshot", sandbox.RemoteErrorKindConflict,
|
||||
"cannot delete template because there are paused sandboxes using it", nil,
|
||||
)
|
||||
|
||||
n, err := fx.svc.PruneSupersededSnapshots(context.Background())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, n)
|
||||
require.Empty(t, fx.provider.deleted)
|
||||
rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, types.SkillSnapshotStateSuperseded, rows[0].State,
|
||||
"an in-use template must stay on the ledger so the next sweep can retry")
|
||||
}
|
||||
|
||||
func TestPruneSupersededSnapshotsTreatsMissingProviderSnapshotAsDeleted(t *testing.T) {
|
||||
fx := newReaperFixture(t)
|
||||
fx.svc.snapshotRetention = time.Hour
|
||||
|
|
|
|||
|
|
@ -43,11 +43,10 @@ const (
|
|||
skillInstallInFlightSkip = 3 * time.Minute
|
||||
|
||||
// skillSnapshotRetention is how long a superseded snapshot stays on the
|
||||
// provider after the pointer has moved. Live sandboxes may still have
|
||||
// been created from it (especially SkillRolloutNewSession); once they
|
||||
// expire, the template is only a billed leftover. Twenty-four hours is
|
||||
// well past every backend's default sandbox TTL. A config that sets a
|
||||
// longer sandbox TTL extends this via snapshotRetentionFor.
|
||||
// provider after the pointer has moved before prune even tries. Paused
|
||||
// session sandboxes can pin the template past this window; prune retries
|
||||
// on Conflict. A config that sets a longer sandbox TTL extends this via
|
||||
// snapshotRetentionFor.
|
||||
skillSnapshotRetention = 24 * time.Hour
|
||||
|
||||
// skillSnapshotTTLMargin is added on top of a config's own sandbox TTL
|
||||
|
|
|
|||
|
|
@ -1123,6 +1123,7 @@ func normalizeCubeError(op string, err error) error {
|
|||
kind = RemoteErrorKindUnavailable
|
||||
}
|
||||
}
|
||||
kind = snapshotDeleteKind(op, kind, err.Error())
|
||||
remoteErr := NewRemoteError(SandboxTypeCube, op, kind, err.Error(), err)
|
||||
remoteErr.StatusCode = status
|
||||
return remoteErr
|
||||
|
|
|
|||
|
|
@ -470,6 +470,14 @@ func TestNormalizeCubeError(t *testing.T) {
|
|||
{"bad gateway", "List", &cubesandbox.APIError{StatusCode: http.StatusBadGateway}, RemoteErrorKindUnavailable},
|
||||
{"deadline", "Exec", context.DeadlineExceeded, RemoteErrorKindTimeout},
|
||||
{"unknown", "List", errors.New("unknown"), RemoteErrorKindInternal},
|
||||
{"delete snapshot in use", "DeleteSnapshot", &cubesandbox.APIError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "cannot delete template x because there are paused sandboxes using it",
|
||||
}, RemoteErrorKindConflict},
|
||||
{"delete snapshot bad id", "DeleteSnapshot", &cubesandbox.APIError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "invalid snapshot id",
|
||||
}, RemoteErrorKindInvalidRequest},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
|
|||
|
|
@ -373,7 +373,7 @@ func dockerError(op string, err error) error {
|
|||
return err
|
||||
}
|
||||
return &RemoteError{
|
||||
Kind: dockerErrorKind(op, err),
|
||||
Kind: snapshotDeleteKind(op, dockerErrorKind(op, err), err.Error()),
|
||||
Provider: SandboxTypeDocker,
|
||||
Op: op,
|
||||
Message: err.Error(),
|
||||
|
|
|
|||
|
|
@ -1253,6 +1253,7 @@ func normalizeE2BError(op string, err error) error {
|
|||
}
|
||||
}
|
||||
}
|
||||
kind = snapshotDeleteKind(op, kind, err.Error())
|
||||
remoteErr := NewRemoteError(SandboxTypeE2B, op, kind, err.Error(), err)
|
||||
remoteErr.StatusCode = status
|
||||
return remoteErr
|
||||
|
|
|
|||
|
|
@ -1391,6 +1391,14 @@ func TestNormalizeE2BError(t *testing.T) {
|
|||
{"connect data loss", "List", connect.NewError(connect.CodeDataLoss, errors.New("data loss")), RemoteErrorKindInternal},
|
||||
{"connect unknown", "List", connect.NewError(connect.CodeUnknown, errors.New("unknown")), RemoteErrorKindInternal},
|
||||
{"unknown", "List", errors.New("mystery"), RemoteErrorKindInternal},
|
||||
{"delete snapshot in use", "DeleteSnapshot", &e2b.Error{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "cannot delete template 'tpl-1' because there are paused sandboxes using it",
|
||||
}, RemoteErrorKindConflict},
|
||||
{"delete snapshot bad id", "DeleteSnapshot", &e2b.Error{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Message: "invalid snapshot id",
|
||||
}, RemoteErrorKindInvalidRequest},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -215,10 +215,24 @@ func (c *langfuseSnapshotClient) DeleteSnapshot(ctx context.Context, snapshotID
|
|||
"snapshot_id": snapshotID,
|
||||
}, nil)
|
||||
err := inner.DeleteSnapshot(ctx, snapshotID)
|
||||
span.Finish(nil, nil, err)
|
||||
out, spanErr := snapshotDeleteSpanResult(err)
|
||||
span.Finish(out, nil, spanErr)
|
||||
return err
|
||||
}
|
||||
|
||||
// snapshotDeleteSpanResult keeps an in-use Conflict off the ERROR status.
|
||||
// Session sandboxes pause against the template they booted from, so the skill
|
||||
// reaper hitting this every few minutes is expected retry, not a fault.
|
||||
func snapshotDeleteSpanResult(err error) (map[string]interface{}, error) {
|
||||
if !IsRemoteConflict(err) {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]interface{}{
|
||||
"deferred": true,
|
||||
"reason": "in_use",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *langfuseSnapshotClient) ListSnapshots(
|
||||
ctx context.Context, sandboxID string,
|
||||
) ([]RemoteSnapshotRef, error) {
|
||||
|
|
|
|||
|
|
@ -61,3 +61,18 @@ func TestTruncateSandboxPreview(t *testing.T) {
|
|||
require.True(t, strings.HasSuffix(got, "…"))
|
||||
require.Equal(t, sandboxSpanPreviewRunes+1, utf8.RuneCountInString(got))
|
||||
}
|
||||
|
||||
func TestSnapshotDeleteSpanResultDefersConflict(t *testing.T) {
|
||||
out, spanErr := snapshotDeleteSpanResult(NewRemoteError(
|
||||
SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict,
|
||||
"paused sandboxes using it", nil,
|
||||
))
|
||||
require.NoError(t, spanErr, "in-use must not mark the Langfuse span ERROR")
|
||||
require.Equal(t, true, out["deferred"])
|
||||
require.Equal(t, "in_use", out["reason"])
|
||||
|
||||
boom := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInternal, "boom", nil)
|
||||
out, spanErr = snapshotDeleteSpanResult(boom)
|
||||
require.Equal(t, boom, spanErr)
|
||||
require.Nil(t, out)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -184,6 +184,37 @@ func IsRemoteInvalidRequest(err error) bool {
|
|||
return remoteKind(err) == RemoteErrorKindInvalidRequest
|
||||
}
|
||||
|
||||
// IsRemoteConflict reports whether the provider refused the call because the
|
||||
// resource is busy (concurrent mutation, snapshot still referenced, …).
|
||||
func IsRemoteConflict(err error) bool {
|
||||
return remoteKind(err) == RemoteErrorKindConflict
|
||||
}
|
||||
|
||||
// snapshotDeleteKind reclassifies a snapshot/template delete that failed
|
||||
// because sandboxes still reference it. E2B returns that as HTTP 400
|
||||
// invalid_request; it is Conflict: the caller did nothing wrong and should
|
||||
// retry after those sandboxes (typically paused) go away.
|
||||
func snapshotDeleteKind(op string, kind RemoteErrorKind, message string) RemoteErrorKind {
|
||||
if op != "DeleteSnapshot" && op != "DeleteTemplate" {
|
||||
return kind
|
||||
}
|
||||
if kind == RemoteErrorKindConflict || snapshotInUseBySandboxes(message) {
|
||||
return RemoteErrorKindConflict
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
func snapshotInUseBySandboxes(message string) bool {
|
||||
msg := strings.ToLower(message)
|
||||
if strings.Contains(msg, "paused sandbox") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(msg, "sandboxes using") {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(msg, "cannot delete template") && strings.Contains(msg, "using it")
|
||||
}
|
||||
|
||||
// IsRemoteDirAlreadyExists reports whether MakeDir failed because the
|
||||
// directory is already present. Cube's envd MakeDir is not mkdir -p: a
|
||||
// directory created by a previous call (or by a shell `mkdir -p` in
|
||||
|
|
|
|||
|
|
@ -177,3 +177,88 @@ func TestIsRemoteDirAlreadyExists(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRemoteConflict(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if IsRemoteConflict(nil) {
|
||||
t.Fatal("nil is not a conflict")
|
||||
}
|
||||
conflict := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict, "busy", nil)
|
||||
if !IsRemoteConflict(conflict) {
|
||||
t.Fatal("Conflict kind must match")
|
||||
}
|
||||
invalid := NewRemoteError(
|
||||
SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInvalidRequest, "bad id", nil,
|
||||
)
|
||||
if IsRemoteConflict(invalid) {
|
||||
t.Fatal("InvalidRequest must not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotDeleteKindPromotesInUseToConflict(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
inUse := "cannot delete template 'upfvuzpq0q6foo1mkpbkl' because there are paused sandboxes using it"
|
||||
jsonBody := `e2b: status 400: {"code":400,"message":"` + inUse + `"}`
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
op string
|
||||
kind RemoteErrorKind
|
||||
msg string
|
||||
want RemoteErrorKind
|
||||
}{
|
||||
{
|
||||
name: "e2b paused sandboxes 400",
|
||||
op: "DeleteSnapshot",
|
||||
kind: RemoteErrorKindInvalidRequest,
|
||||
msg: inUse,
|
||||
want: RemoteErrorKindConflict,
|
||||
},
|
||||
{
|
||||
name: "json-wrapped e2b 400",
|
||||
op: "DeleteSnapshot",
|
||||
kind: RemoteErrorKindInvalidRequest,
|
||||
msg: jsonBody,
|
||||
want: RemoteErrorKindConflict,
|
||||
},
|
||||
{
|
||||
name: "cube delete template",
|
||||
op: "DeleteTemplate",
|
||||
kind: RemoteErrorKindInvalidRequest,
|
||||
msg: "cannot delete template x because there are sandboxes using it",
|
||||
want: RemoteErrorKindConflict,
|
||||
},
|
||||
{
|
||||
name: "already conflict stays conflict",
|
||||
op: "DeleteSnapshot",
|
||||
kind: RemoteErrorKindConflict,
|
||||
msg: "image is in use",
|
||||
want: RemoteErrorKindConflict,
|
||||
},
|
||||
{
|
||||
name: "generic 400 stays invalid",
|
||||
op: "DeleteSnapshot",
|
||||
kind: RemoteErrorKindInvalidRequest,
|
||||
msg: "invalid snapshot id",
|
||||
want: RemoteErrorKindInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "other ops are unchanged",
|
||||
op: "Create",
|
||||
kind: RemoteErrorKindInvalidRequest,
|
||||
msg: inUse,
|
||||
want: RemoteErrorKindInvalidRequest,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := snapshotDeleteKind(tt.op, tt.kind, tt.msg); got != tt.want {
|
||||
t.Fatalf("snapshotDeleteKind() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue