diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.go index e6f70f69..fada45db 100644 --- a/internal/application/service/tenant_skill_reaper.go +++ b/internal/application/service/tenant_skill_reaper.go @@ -422,10 +422,10 @@ func configuredSandboxTTL(cfg *types.TenantSandboxConfig) time.Duration { // image is never touched, nor is anything the ledger does not name: extras // belong to other environments on a shared provider account. // -// A live sandbox does not need the template it was created from in order to -// keep running, so the only reason to wait is in-flight creates that resolved -// the previous pointer. Twenty-four hours is far past every backend's default -// TTL; a config that sets a longer one extends the wait. +// Retention is a lower bound, not a guarantee the provider will accept the +// delete. Session sandboxes pause instead of dying when idle, and a paused +// sandbox still pins its template; a Conflict is left on the ledger for the +// next sweep. func (s *TenantSkillService) PruneSupersededSnapshots(ctx context.Context) (int, error) { if s == nil || s.skills == nil { return 0, nil @@ -497,6 +497,12 @@ func (s *TenantSkillService) pruneConfigSnapshots( continue } if err := deleter.DeleteSnapshot(ctx, row.SnapshotID); err != nil && !sandbox.IsRemoteNotFound(err) { + if sandbox.IsRemoteConflict(err) { + logger.Infof(ctx, + "[skill] snapshot %s still in use; leaving it until sandboxes release it: %s", + row.SnapshotID, sandbox.RemoteErrorDiagnostics(err)) + continue + } logger.Warnf(ctx, "[skill] delete superseded snapshot %s failed: %v", row.SnapshotID, err) continue } @@ -568,6 +574,12 @@ func (s *TenantSkillService) reapAbandonedBuilds( continue } if err := deleter.DeleteSnapshot(ctx, found); err != nil && !sandbox.IsRemoteNotFound(err) { + if sandbox.IsRemoteConflict(err) { + logger.Infof(ctx, + "[skill] abandoned build %s of config %s still in use; leaving it until sandboxes release it: %s", + found, cfg.ID, sandbox.RemoteErrorDiagnostics(err)) + continue + } logger.Warnf(ctx, "[skill] delete abandoned build %s of config %s failed: %v", found, cfg.ID, err) continue diff --git a/internal/application/service/tenant_skill_reaper_test.go b/internal/application/service/tenant_skill_reaper_test.go index f6965751..c56f34ee 100644 --- a/internal/application/service/tenant_skill_reaper_test.go +++ b/internal/application/service/tenant_skill_reaper_test.go @@ -390,6 +390,28 @@ func TestPruneSupersededSnapshotsLeavesTheRowWhenDeleteFails(t *testing.T) { "a failed provider delete must not be recorded as deleted") } +func TestPruneSupersededSnapshotsRetriesWhenSnapshotStillInUse(t *testing.T) { + fx := newReaperFixture(t) + fx.svc.snapshotRetention = time.Hour + old := fx.now.Add(-2 * time.Hour) + fx.live("snap-live") + fx.superseded("sk-1", "snap-old", "", old) + fx.provider.deleteErr = sandbox.NewRemoteError( + sandbox.SandboxTypeE2B, "DeleteSnapshot", sandbox.RemoteErrorKindConflict, + "cannot delete template because there are paused sandboxes using it", nil, + ) + + n, err := fx.svc.PruneSupersededSnapshots(context.Background()) + + require.NoError(t, err) + require.Zero(t, n) + require.Empty(t, fx.provider.deleted) + rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1") + require.NoError(t, err) + require.Equal(t, types.SkillSnapshotStateSuperseded, rows[0].State, + "an in-use template must stay on the ledger so the next sweep can retry") +} + func TestPruneSupersededSnapshotsTreatsMissingProviderSnapshotAsDeleted(t *testing.T) { fx := newReaperFixture(t) fx.svc.snapshotRetention = time.Hour diff --git a/internal/application/service/tenant_skill_service.go b/internal/application/service/tenant_skill_service.go index 247775cc..6dabe2e6 100644 --- a/internal/application/service/tenant_skill_service.go +++ b/internal/application/service/tenant_skill_service.go @@ -43,11 +43,10 @@ const ( skillInstallInFlightSkip = 3 * time.Minute // skillSnapshotRetention is how long a superseded snapshot stays on the - // provider after the pointer has moved. Live sandboxes may still have - // been created from it (especially SkillRolloutNewSession); once they - // expire, the template is only a billed leftover. Twenty-four hours is - // well past every backend's default sandbox TTL. A config that sets a - // longer sandbox TTL extends this via snapshotRetentionFor. + // provider after the pointer has moved before prune even tries. Paused + // session sandboxes can pin the template past this window; prune retries + // on Conflict. A config that sets a longer sandbox TTL extends this via + // snapshotRetentionFor. skillSnapshotRetention = 24 * time.Hour // skillSnapshotTTLMargin is added on top of a config's own sandbox TTL diff --git a/internal/sandbox/cube_remote_client.go b/internal/sandbox/cube_remote_client.go index c10de2fb..562ae6c2 100644 --- a/internal/sandbox/cube_remote_client.go +++ b/internal/sandbox/cube_remote_client.go @@ -1123,6 +1123,7 @@ func normalizeCubeError(op string, err error) error { kind = RemoteErrorKindUnavailable } } + kind = snapshotDeleteKind(op, kind, err.Error()) remoteErr := NewRemoteError(SandboxTypeCube, op, kind, err.Error(), err) remoteErr.StatusCode = status return remoteErr diff --git a/internal/sandbox/cube_remote_client_test.go b/internal/sandbox/cube_remote_client_test.go index c5eca910..0e487bcd 100644 --- a/internal/sandbox/cube_remote_client_test.go +++ b/internal/sandbox/cube_remote_client_test.go @@ -470,6 +470,14 @@ func TestNormalizeCubeError(t *testing.T) { {"bad gateway", "List", &cubesandbox.APIError{StatusCode: http.StatusBadGateway}, RemoteErrorKindUnavailable}, {"deadline", "Exec", context.DeadlineExceeded, RemoteErrorKindTimeout}, {"unknown", "List", errors.New("unknown"), RemoteErrorKindInternal}, + {"delete snapshot in use", "DeleteSnapshot", &cubesandbox.APIError{ + StatusCode: http.StatusBadRequest, + Message: "cannot delete template x because there are paused sandboxes using it", + }, RemoteErrorKindConflict}, + {"delete snapshot bad id", "DeleteSnapshot", &cubesandbox.APIError{ + StatusCode: http.StatusBadRequest, + Message: "invalid snapshot id", + }, RemoteErrorKindInvalidRequest}, } for _, tt := range tests { diff --git a/internal/sandbox/docker_engine.go b/internal/sandbox/docker_engine.go index c5e07d13..0e08558b 100644 --- a/internal/sandbox/docker_engine.go +++ b/internal/sandbox/docker_engine.go @@ -373,7 +373,7 @@ func dockerError(op string, err error) error { return err } return &RemoteError{ - Kind: dockerErrorKind(op, err), + Kind: snapshotDeleteKind(op, dockerErrorKind(op, err), err.Error()), Provider: SandboxTypeDocker, Op: op, Message: err.Error(), diff --git a/internal/sandbox/e2b_remote_client.go b/internal/sandbox/e2b_remote_client.go index f835044f..1d93906c 100644 --- a/internal/sandbox/e2b_remote_client.go +++ b/internal/sandbox/e2b_remote_client.go @@ -1253,6 +1253,7 @@ func normalizeE2BError(op string, err error) error { } } } + kind = snapshotDeleteKind(op, kind, err.Error()) remoteErr := NewRemoteError(SandboxTypeE2B, op, kind, err.Error(), err) remoteErr.StatusCode = status return remoteErr diff --git a/internal/sandbox/e2b_remote_client_test.go b/internal/sandbox/e2b_remote_client_test.go index 09737b58..e2b43b88 100644 --- a/internal/sandbox/e2b_remote_client_test.go +++ b/internal/sandbox/e2b_remote_client_test.go @@ -1391,6 +1391,14 @@ func TestNormalizeE2BError(t *testing.T) { {"connect data loss", "List", connect.NewError(connect.CodeDataLoss, errors.New("data loss")), RemoteErrorKindInternal}, {"connect unknown", "List", connect.NewError(connect.CodeUnknown, errors.New("unknown")), RemoteErrorKindInternal}, {"unknown", "List", errors.New("mystery"), RemoteErrorKindInternal}, + {"delete snapshot in use", "DeleteSnapshot", &e2b.Error{ + StatusCode: http.StatusBadRequest, + Message: "cannot delete template 'tpl-1' because there are paused sandboxes using it", + }, RemoteErrorKindConflict}, + {"delete snapshot bad id", "DeleteSnapshot", &e2b.Error{ + StatusCode: http.StatusBadRequest, + Message: "invalid snapshot id", + }, RemoteErrorKindInvalidRequest}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/sandbox/langfuse.go b/internal/sandbox/langfuse.go index 8cab1e67..363b653b 100644 --- a/internal/sandbox/langfuse.go +++ b/internal/sandbox/langfuse.go @@ -215,10 +215,24 @@ func (c *langfuseSnapshotClient) DeleteSnapshot(ctx context.Context, snapshotID "snapshot_id": snapshotID, }, nil) err := inner.DeleteSnapshot(ctx, snapshotID) - span.Finish(nil, nil, err) + out, spanErr := snapshotDeleteSpanResult(err) + span.Finish(out, nil, spanErr) return err } +// snapshotDeleteSpanResult keeps an in-use Conflict off the ERROR status. +// Session sandboxes pause against the template they booted from, so the skill +// reaper hitting this every few minutes is expected retry, not a fault. +func snapshotDeleteSpanResult(err error) (map[string]interface{}, error) { + if !IsRemoteConflict(err) { + return nil, err + } + return map[string]interface{}{ + "deferred": true, + "reason": "in_use", + }, nil +} + func (c *langfuseSnapshotClient) ListSnapshots( ctx context.Context, sandboxID string, ) ([]RemoteSnapshotRef, error) { diff --git a/internal/sandbox/langfuse_test.go b/internal/sandbox/langfuse_test.go index 84c11d27..8687bbf1 100644 --- a/internal/sandbox/langfuse_test.go +++ b/internal/sandbox/langfuse_test.go @@ -61,3 +61,18 @@ func TestTruncateSandboxPreview(t *testing.T) { require.True(t, strings.HasSuffix(got, "…")) require.Equal(t, sandboxSpanPreviewRunes+1, utf8.RuneCountInString(got)) } + +func TestSnapshotDeleteSpanResultDefersConflict(t *testing.T) { + out, spanErr := snapshotDeleteSpanResult(NewRemoteError( + SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict, + "paused sandboxes using it", nil, + )) + require.NoError(t, spanErr, "in-use must not mark the Langfuse span ERROR") + require.Equal(t, true, out["deferred"]) + require.Equal(t, "in_use", out["reason"]) + + boom := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInternal, "boom", nil) + out, spanErr = snapshotDeleteSpanResult(boom) + require.Equal(t, boom, spanErr) + require.Nil(t, out) +} diff --git a/internal/sandbox/remote_errors.go b/internal/sandbox/remote_errors.go index b5041240..b89eec0d 100644 --- a/internal/sandbox/remote_errors.go +++ b/internal/sandbox/remote_errors.go @@ -184,6 +184,37 @@ func IsRemoteInvalidRequest(err error) bool { return remoteKind(err) == RemoteErrorKindInvalidRequest } +// IsRemoteConflict reports whether the provider refused the call because the +// resource is busy (concurrent mutation, snapshot still referenced, …). +func IsRemoteConflict(err error) bool { + return remoteKind(err) == RemoteErrorKindConflict +} + +// snapshotDeleteKind reclassifies a snapshot/template delete that failed +// because sandboxes still reference it. E2B returns that as HTTP 400 +// invalid_request; it is Conflict: the caller did nothing wrong and should +// retry after those sandboxes (typically paused) go away. +func snapshotDeleteKind(op string, kind RemoteErrorKind, message string) RemoteErrorKind { + if op != "DeleteSnapshot" && op != "DeleteTemplate" { + return kind + } + if kind == RemoteErrorKindConflict || snapshotInUseBySandboxes(message) { + return RemoteErrorKindConflict + } + return kind +} + +func snapshotInUseBySandboxes(message string) bool { + msg := strings.ToLower(message) + if strings.Contains(msg, "paused sandbox") { + return true + } + if strings.Contains(msg, "sandboxes using") { + return true + } + return strings.Contains(msg, "cannot delete template") && strings.Contains(msg, "using it") +} + // IsRemoteDirAlreadyExists reports whether MakeDir failed because the // directory is already present. Cube's envd MakeDir is not mkdir -p: a // directory created by a previous call (or by a shell `mkdir -p` in diff --git a/internal/sandbox/remote_errors_test.go b/internal/sandbox/remote_errors_test.go index cc7df831..0ad3dee9 100644 --- a/internal/sandbox/remote_errors_test.go +++ b/internal/sandbox/remote_errors_test.go @@ -177,3 +177,88 @@ func TestIsRemoteDirAlreadyExists(t *testing.T) { }) } } + +func TestIsRemoteConflict(t *testing.T) { + t.Parallel() + + if IsRemoteConflict(nil) { + t.Fatal("nil is not a conflict") + } + conflict := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict, "busy", nil) + if !IsRemoteConflict(conflict) { + t.Fatal("Conflict kind must match") + } + invalid := NewRemoteError( + SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInvalidRequest, "bad id", nil, + ) + if IsRemoteConflict(invalid) { + t.Fatal("InvalidRequest must not match") + } +} + +func TestSnapshotDeleteKindPromotesInUseToConflict(t *testing.T) { + t.Parallel() + + inUse := "cannot delete template 'upfvuzpq0q6foo1mkpbkl' because there are paused sandboxes using it" + jsonBody := `e2b: status 400: {"code":400,"message":"` + inUse + `"}` + + tests := []struct { + name string + op string + kind RemoteErrorKind + msg string + want RemoteErrorKind + }{ + { + name: "e2b paused sandboxes 400", + op: "DeleteSnapshot", + kind: RemoteErrorKindInvalidRequest, + msg: inUse, + want: RemoteErrorKindConflict, + }, + { + name: "json-wrapped e2b 400", + op: "DeleteSnapshot", + kind: RemoteErrorKindInvalidRequest, + msg: jsonBody, + want: RemoteErrorKindConflict, + }, + { + name: "cube delete template", + op: "DeleteTemplate", + kind: RemoteErrorKindInvalidRequest, + msg: "cannot delete template x because there are sandboxes using it", + want: RemoteErrorKindConflict, + }, + { + name: "already conflict stays conflict", + op: "DeleteSnapshot", + kind: RemoteErrorKindConflict, + msg: "image is in use", + want: RemoteErrorKindConflict, + }, + { + name: "generic 400 stays invalid", + op: "DeleteSnapshot", + kind: RemoteErrorKindInvalidRequest, + msg: "invalid snapshot id", + want: RemoteErrorKindInvalidRequest, + }, + { + name: "other ops are unchanged", + op: "Create", + kind: RemoteErrorKindInvalidRequest, + msg: inUse, + want: RemoteErrorKindInvalidRequest, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := snapshotDeleteKind(tt.op, tt.kind, tt.msg); got != tt.want { + t.Fatalf("snapshotDeleteKind() = %q, want %q", got, tt.want) + } + }) + } +}