fix(automation): enforce execution identity, cancellation, and scheduling limits (#4506)

* fix(automation): preserve execution identity and scheduler guarantees

* fix(ci): satisfy lint checks for automation regressions
main
lyingbug 2026-10-20 13:00:12 +08:00 committed by GitHub
parent 8dab7550f5
commit a8eb7e28fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 695 additions and 42 deletions

View File

@ -184,12 +184,37 @@ func (s *AgentCollaboration) prune(ctx context.Context) error {
type delegatedConfig struct {
Agent types.CustomAgent `json:"agent"`
Access types.AssistantAccessLevel `json:"access"`
// Durable execution identity follows children across scheduler instances
// and approval resumes; it must not turn an automation into a human turn.
ActingVia string `json:"acting_via,omitempty"`
Mode types.RunMode `json:"mode,omitempty"`
// AssistantParent records that the delegating run was the WeKnora
// assistant, whose level stays under the workspace's assistant ceiling.
AssistantParent bool `json:"assistant_parent,omitempty"`
Inherited *inheritedAgentConfig `json:"inherited,omitempty"`
}
func (d *delegatedConfig) captureExecution(ctx context.Context) {
d.ActingVia = types.ActingViaFromContext(ctx)
d.Mode = types.RunModeAttended
if mode, ok := ctx.Value(types.RunModeContextKey).(types.RunMode); ok {
d.Mode = mode.Normalize()
}
if d.ActingVia != "" {
d.Mode = types.RunModeDeferred
}
}
func (d *delegatedConfig) runMode() types.RunMode {
if d.ActingVia != "" {
return types.RunModeDeferred
}
if d.Mode == "" {
return types.RunModeAttended // snapshots created before mode was stored
}
return d.Mode.Normalize()
}
// level is the child's access level: the delegating run's, capped by the
// child agent's own ceiling and, under an assistant parent, by the
// workspace's current assistant ceiling.
@ -222,7 +247,9 @@ func (s *AgentCollaboration) resolveDelegated(
id == types.BuiltinSkillInstallerID {
return "", errors.New("only same-workspace reasoning agents without management privileges may be delegated")
}
data, err := json.Marshal(delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent})
saved := delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent}
saved.captureExecution(ctx)
data, err := json.Marshal(saved)
return string(data), err
}
@ -377,7 +404,11 @@ func (s *AgentCollaboration) workerContext(ctx context.Context, scope collaborat
ctx = context.WithValue(ctx, types.TenantRoleContextKey, membership.Role)
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
ctx = types.WithCaller(ctx, types.Caller{TenantID: scope.TenantID, UserID: scope.UserID, Role: membership.Role})
ctx = types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID})
principal := types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID}
if previous, ok := types.PrincipalFromContext(ctx); ok && previous.ID == scope.UserID {
principal.ActingVia = previous.ActingVia
}
ctx = types.WithPrincipal(ctx, principal)
return ctx, nil
}
@ -397,6 +428,10 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (col
if err = json.Unmarshal([]byte(w.Member.ConfigJSON), &saved); err != nil {
return result, err
}
ctx = types.WithPrincipal(ctx, types.Principal{
Type: types.PrincipalWebUser, ID: w.Scope.UserID, ActingVia: saved.ActingVia,
})
ctx = types.WithRunMode(ctx, saved.runMode())
// Both modes pin configuration, but reauthorize the source before each turn.
if saved.Inherited == nil {
if _, err = s.resolve(ctx, w.Member.AgentID, types.AssistantAccessReadOnly); err != nil {
@ -610,7 +645,7 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (col
ctx,
&types.QARequest{
Resume: resume,
Mode: types.RunModeAttended,
Mode: saved.runMode(),
GrantSessionID: w.Scope.SessionID,
Suspendable: true,
Session: session,

View File

@ -75,10 +75,12 @@ func inheritedChildSnapshot(ctx context.Context, cfg *types.AgentConfig, toolNam
}
}
// Marshal immediately: later changes to parent config cannot change queued work.
data, err := json.Marshal(delegatedConfig{
saved := delegatedConfig{
Agent: *source.Agent, Access: cfg.AssistantAccess, AssistantParent: cfg.OperationsAgentID != "",
Inherited: inherited,
})
}
saved.captureExecution(ctx)
data, err := json.Marshal(saved)
return string(data), err
}

View File

@ -223,6 +223,11 @@ func (s *AgentCollaboration) resumeParent(ctx context.Context, b *collaboration.
if err = json.Unmarshal(snap.Request, &saved); err != nil {
return err
}
if saved.ActingVia != "" {
// Automation leases, cancellation and duration belong to their own
// scheduler. Never revive one through ordinary parent recovery.
return errors.New("scheduler-owned parent cannot resume through collaboration recovery")
}
if saved.AgentID == "" || saved.SharedAgentReadOnly {
return errors.New("invalid parent recovery identity")
}

View File

@ -80,6 +80,8 @@ type AutomationService struct {
// Events publishes a run to its message's event stream; see
// AgentCollaboration.ChildEvents.
Events func(context.Context, *types.Session, *types.Message, *event.EventBus) func(context.Context) error
// StopDelegation fences children and discards recoverable parent waits.
StopDelegation func(context.Context, uint64, string, string, string) error
cfg AutomationConfig
instance string
@ -621,16 +623,37 @@ func (s *AutomationService) cancelRun(ctx context.Context, run *types.Automation
Update("cancel_requested", true).Error; err != nil {
return err
}
stopErr := s.stopRunDelegation(ctx, run)
s.mu.Lock()
cancel := s.running[run.ID]
s.mu.Unlock()
if cancel != nil {
cancel()
}
return stopErr
}
return nil
}
func (s *AutomationService) stopRunDelegation(ctx context.Context, run *types.AutomationRun) error {
if s.StopDelegation == nil {
return nil
}
// The caller may hold the row from before the executor created its turn.
var current types.AutomationRun
if err := s.db.WithContext(ctx).Where("id = ?", run.ID).Take(&current).Error; err != nil {
return err
}
if current.SessionID == "" || current.AssistantMessageID == "" {
return nil
}
a, err := s.automationOf(ctx, &current)
if err != nil {
return err
}
return s.StopDelegation(ctx, a.TenantID, a.OwnerUserID, current.SessionID, current.AssistantMessageID)
}
// closePrompt ends the open prompt id a run waits on and completes the
// run's message, as the prompt reaper would.
func (s *AutomationService) closePrompt(ctx context.Context, id, status, reason string) {

View File

@ -0,0 +1,462 @@
package service
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/agent/collaboration"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
func TestAutomationDelegatedExecutionKeepsIdentity(t *testing.T) {
for _, inherited := range []bool{false, true} {
t.Run(fmt.Sprintf("inherited=%t", inherited), func(t *testing.T) {
svc, agents, _, ctx := collaborationService(t)
via := types.AutomationActingVia("scheduled-task")
parent := types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: via})
parent = types.WithRunMode(parent, types.RunModeDeferred)
var snapshot string
var err error
if inherited {
parent = context.WithValue(parent, childSourceKey{}, childSource{Agent: agents.agent, ModelID: "model"})
snapshot, err = inheritedChildSnapshot(
parent,
&types.AgentConfig{AssistantAccess: types.AssistantAccessAutoWrite},
nil,
)
} else {
snapshot, err = svc.resolve(parent, "a", types.AssistantAccessAutoWrite)
}
require.NoError(t, err)
scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"}
require.NoError(t, svc.Runtime.Store.Ensure(ctx, scope))
_, err = svc.Runtime.Spawn(ctx, scope, "spawn", "child", "a", snapshot, "work")
require.NoError(t, err)
require.NoError(t, svc.Runtime.Store.Change(ctx, scope, func(g *collaboration.Group) error {
r := g.Members[0].Runs[0]
r.State = "running"
r.Owner = "worker"
r.Lease = time.Now().Add(time.Minute)
return nil
}))
called := false
svc.Bind(
&collaborationSessions{
run: func(child context.Context, req *types.QARequest, bus *event.EventBus) error {
called = true
require.Equal(t, via, types.ActingViaFromContext(child))
require.Equal(t, types.RunModeDeferred, req.Mode)
require.False(t, req.Mode.Attended(), "ask_user must stay unavailable")
_, _, err := automationOwner(child)
require.Error(t, err, "delegation must not bypass the automation management guard")
return complete(child, bus, event.AgentCompleteData{FinalAnswer: "done"})
},
},
)
g, err := svc.Runtime.Store.Read(ctx, scope)
require.NoError(t, err)
// A scheduler starts from its own context, not the parent's request context.
_, err = svc.run(
context.Background(),
collaboration.Work{Scope: scope, Member: *g.Members[0], Run: *g.Members[0].Runs[0]},
)
require.NoError(t, err)
require.True(t, called)
})
}
}
func TestAutomationCancelStopsDelegationAndRecovery(t *testing.T) {
f := newAutomationFixture(t)
require.NoError(t, f.db.AutoMigrate(&collaboration.Record{}))
delegated := NewAgentCollaboration(f.db, f.agents, f.messages, f.users, f.members, f.svc.tenants)
f.svc.StopDelegation = delegated.StopOwned
a := f.create(t, types.AutomationPatch{})
queued, err := f.svc.RunNow(f.ctx, a.ID)
require.NoError(t, err)
entered := make(chan collaboration.Scope, 1)
f.sessions.run = func(ctx context.Context, req *types.QARequest, _ *event.EventBus) error {
scope := collaboration.Scope{
TenantID: automationTenant,
UserID: "u",
SessionID: req.Session.ID,
ID: req.AssistantMessageID,
}
b, err := delegated.Runtime.AcquireParent(ctx, scope)
if err != nil {
return err
}
_, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn", "child", "a", "{}", "work")
if err != nil {
return err
}
_, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn-running", "running_child", "a", "{}", "work")
if err != nil {
return err
}
if err = delegated.Runtime.Store.Change(b.Context(ctx), scope, func(g *collaboration.Group) error {
r := g.Members[1].Runs[0]
r.State, r.Owner, r.Lease = "running", "remote-worker", time.Now().Add(time.Minute)
return nil
}); err != nil {
return err
}
saved := resumableAgentRequest(ctx, req)
data, err := json.Marshal(
agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, saved), CollaborationWait: true},
)
if err != nil {
return err
}
if err = b.SaveWait(ctx, data, time.Now().Add(time.Minute)); err != nil {
return err
}
entered <- scope
<-ctx.Done()
// Exercise the same cleanup as an engine cancelled while in wait_agent.
b.End(true)
return ctx.Err()
}
require.NoError(t, f.svc.startQueued(f.ctx, f.now))
var scope collaboration.Scope
select {
case scope = <-entered:
case <-time.After(5 * time.Second):
t.Fatal("run did not start")
}
_, err = f.svc.CancelRun(f.ctx, queued.ID)
require.NoError(t, err)
done := f.settleRun(t, queued.ID)
require.Equal(t, types.AutomationRunCancelled, done.Status)
g, err := delegated.Runtime.Store.Read(f.ctx, scope)
require.NoError(t, err)
require.NotEqual(t, "active", g.Status)
require.Nil(t, g.ParentWait, "cancelled checkpoints must not be recovered")
require.True(t, g.Members[0].Runs[0].CancelRequested)
require.Equal(t, "cancelled", g.Members[0].Runs[0].State)
require.True(t, g.Members[1].Runs[0].CancelRequested)
require.ErrorIs(t, delegated.Runtime.CheckOwner(f.ctx, collaboration.Work{
Scope: scope, Member: *g.Members[1], Run: *g.Members[1].Runs[0],
}), collaboration.ErrConflict, "a remote child must lose permission to execute tools immediately")
}
func mustJSON(t *testing.T, v any) json.RawMessage {
t.Helper()
data, err := json.Marshal(v)
require.NoError(t, err)
return data
}
func TestAutomationParentRecoveryDoesNotEscapeScheduler(t *testing.T) {
svc, _, messages, ctx := collaborationService(t)
scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"}
b, err := svc.Runtime.AcquireParent(ctx, scope)
require.NoError(t, err)
t.Cleanup(func() { b.End(false) })
require.NoError(t, messages.UpdateMessage(ctx, &types.Message{ID: scope.ID, SessionID: scope.SessionID}))
svc.streams = &collaborationStreams{}
svc.ParentEvents = func(
context.Context, *types.Session, *types.Message, *event.EventBus,
) func(context.Context) error {
return func(context.Context) error { return nil }
}
called := false
svc.Bind(
&collaborationSessions{
run: func(context.Context, *types.QARequest, *event.EventBus) error { called = true; return nil },
},
)
ctx = types.WithPrincipal(
ctx,
types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: types.AutomationActingVia("a")},
)
req := resumableAgentRequest(
ctx,
&types.QARequest{CustomAgent: &types.CustomAgent{ID: "a"}, Mode: types.RunModeDeferred},
)
snapshot := mustJSON(
t,
agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, req), CollaborationWait: true},
)
err = svc.resumeParent(context.Background(), b, snapshot)
require.ErrorContains(t, err, "scheduler-owned parent")
require.False(t, called)
}
func TestAutomationStartSerializesDifferentRunsAcrossInstances(t *testing.T) {
f := newAutomationFixture(t)
a := f.create(t, types.AutomationPatch{OverlapPolicy: ptr(types.AutomationPolicyQueue)})
rows := []*types.AutomationRun{
{
AutomationID: a.ID,
TenantID: automationTenant,
ScheduledAt: f.now.Add(-time.Hour),
TriggerKind: types.AutomationRunSchedule,
Status: types.AutomationRunQueued,
Resume: true,
},
{
AutomationID: a.ID,
TenantID: automationTenant,
ScheduledAt: f.now,
TriggerKind: types.AutomationRunSchedule,
Status: types.AutomationRunQueued,
},
}
for _, r := range rows {
require.NoError(t, f.db.Create(r).Error)
}
// A separate connection pool represents another scheduler process.
db, err := gorm.Open(
sqlite.Open(f.db.Dialector.(*sqlite.Dialector).DSN),
&gorm.Config{Logger: logger.Default.LogMode(logger.Silent)},
)
require.NoError(t, err)
raw, err := db.DB()
require.NoError(t, err)
t.Cleanup(func() { _ = raw.Close() })
other := &AutomationService{db: db, instance: "second-instance"}
arrived := make(chan struct{}, 2)
release := make(chan struct{})
for _, db := range []*gorm.DB{f.db, db} {
require.NoError(t, db.Callback().Query().After("gorm:query").Register("admission_barrier", func(db *gorm.DB) {
if n, ok := db.Statement.Dest.(*int64); ok && *n == 0 && db.Statement.Table == "automation_runs" &&
strings.Contains(db.Statement.SQL.String(), "count(*)") {
arrived <- struct{}{}
<-release
}
}))
}
var wg sync.WaitGroup
wg.Add(2)
started := make([]bool, 2)
errs := make([]error, 2)
go func() { defer wg.Done(); started[0], errs[0] = f.svc.startRun(f.ctx, rows[0], f.now) }()
select {
case <-arrived:
case <-time.After(5 * time.Second):
close(release)
t.Fatal("first admission did not reach check")
}
go func() { defer wg.Done(); started[1], errs[1] = other.startRun(f.ctx, rows[1], f.now) }()
// The old implementation lets both count queries observe zero. With the
// automation lock the second query cannot run until the first commits.
select {
case <-arrived:
case <-time.After(100 * time.Millisecond):
}
close(release)
wg.Wait()
require.NoError(t, errs[0])
require.NoError(t, errs[1])
require.True(t, started[0])
require.False(t, started[1])
var active int64
require.NoError(
t,
f.db.Model(&types.AutomationRun{}).
Where("automation_id = ? AND status = ?", a.ID, types.AutomationRunRunning).
Count(&active).
Error,
)
require.EqualValues(t, 1, active)
}
func TestAutomationQueuedRunsSkipBlockedHead(t *testing.T) {
f := newAutomationFixture(t)
for i := 0; i < 16; i++ {
a := f.create(
t,
types.AutomationPatch{
Name: ptr(fmt.Sprintf("blocked%d", i)),
OverlapPolicy: ptr(types.AutomationPolicyQueue),
},
)
for j, status := range []string{types.AutomationRunAwaitingApproval, types.AutomationRunQueued} {
require.NoError(
t,
f.db.Create(
&types.AutomationRun{
AutomationID: a.ID,
TenantID: automationTenant,
ScheduledAt: f.now.Add(time.Duration(j-2) * time.Hour),
TriggerKind: types.AutomationRunSchedule,
Status: status,
},
).Error,
)
}
}
a := f.create(t, types.AutomationPatch{Name: ptr("runnable")})
r, err := f.svc.RunNow(f.ctx, a.ID)
require.NoError(t, err)
require.NoError(t, f.svc.startQueued(context.Background(), f.now))
require.Eventually(t, func() bool {
var row types.AutomationRun
return f.db.First(&row, "id = ?", r.ID).Error == nil && row.Status == types.AutomationRunSucceeded &&
len(f.svc.slots) == 0
}, 5*time.Second, 10*time.Millisecond)
}
func TestAutomationApprovalReconciliationVisitsEveryBatch(t *testing.T) {
f := newAutomationFixture(t)
f.svc.cfg.MaxPerTenant = 200
var expired, answered *types.AutomationRun
for i := 0; i < 102; i++ {
timeout := 10080
updated := f.now.Add(-time.Hour)
if i == 100 {
timeout = 5
updated = f.now.Add(-10 * time.Minute)
}
a := f.create(
t,
types.AutomationPatch{Name: ptr(fmt.Sprintf("approval%d", i)), ApprovalTimeoutMinutes: ptr(timeout)},
)
status := types.AgentPromptSuspended
if i == 0 || i == 101 {
status = types.AgentPromptAnswered
}
p := &types.AgentPrompt{ID: fmt.Sprintf("p%d", i), Status: status}
require.NoError(t, f.broker.Store().Create(f.ctx, p))
r := &types.AutomationRun{
ID: fmt.Sprintf("run-%03d", i),
AutomationID: a.ID,
TenantID: automationTenant,
ScheduledAt: updated,
TriggerKind: types.AutomationRunSchedule,
Status: types.AutomationRunAwaitingApproval,
PendingPromptID: p.ID,
UpdatedAt: updated,
}
require.NoError(t, f.db.Create(r).Error)
if i == 100 {
expired = r
}
if i == 101 {
answered = r
}
}
require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now))
require.NoError(t, f.db.First(expired, "id = ?", expired.ID).Error)
require.Equal(t, types.AutomationRunFailed, expired.Status)
require.NoError(t, f.db.First(answered, "id = ?", answered.ID).Error)
require.Equal(t, types.AutomationRunQueued, answered.Status)
require.True(t, answered.Resume)
}
func TestAutomationApprovalResumeUsesRemainingExecutionBudget(t *testing.T) {
f := newAutomationFixture(t)
f.svc.cfg.DefaultMaxDuration = time.Second
a := f.create(t, types.AutomationPatch{})
f.suspendOnApproval(t, "op:x")
original := f.sessions.run
f.sessions.run = func(ctx context.Context, req *types.QARequest, bus *event.EventBus) error {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(600 * time.Millisecond):
}
return original(ctx, req, bus)
}
queued, err := f.svc.RunNow(f.ctx, a.ID)
require.NoError(t, err)
suspended := f.settleRun(t, queued.ID)
require.Equal(t, types.AutomationRunAwaitingApproval, suspended.Status)
// Approval waiting is free, even across service restarts.
f.now = f.now.Add(12 * time.Hour)
restarted := NewAutomationService(
f.db,
f.agents,
f.users,
f.members,
f.svc.tenants,
f.messages,
f.svc.grants,
f.svc.notices,
)
restarted.Configure(AutomationConfig{DefaultMaxDuration: time.Second})
restarted.now = func() time.Time { return f.now }
restarted.Bind(f.sessions, f.broker, nil, nil)
f.svc = restarted
ok, err := f.broker.Store().Transition(f.ctx, suspended.PendingPromptID, types.AgentPromptTransition{
From: []string{types.AgentPromptSuspended},
To: types.AgentPromptAnswered,
Response: types.JSON(`{"approved":true}`),
})
require.NoError(t, err)
require.True(t, ok)
resumed := f.settleRun(t, queued.ID)
require.Equal(t, types.AutomationRunFailed, resumed.Status)
require.Contains(t, resumed.Error, "longer than 1s")
require.NotNil(t, f.sessions.last().req.Resume, "waiting time must not exhaust the execution budget")
}
func TestAutomationStaleFinalizerDoesNotCancelResumedDelegation(t *testing.T) {
f := newAutomationFixture(t)
a := f.create(t, types.AutomationPatch{})
r, err := f.svc.RunNow(f.ctx, a.ID)
require.NoError(t, err)
require.NoError(t, f.db.Model(r).Updates(map[string]any{
"status": types.AutomationRunRunning, "session_id": "session", "assistant_message_id": "answer",
}).Error)
stopped := false
f.svc.StopDelegation = func(context.Context, uint64, string, string, string) error { stopped = true; return nil }
ok, err := f.svc.finishRun(
f.ctx,
r,
[]string{types.AutomationRunAwaitingApproval},
types.AutomationRunFailed,
"stale timeout",
nil,
)
require.NoError(t, err)
require.False(t, ok)
require.False(t, stopped)
ok, err = f.svc.finishRun(
f.ctx,
r,
[]string{types.AutomationRunRunning},
types.AutomationRunNeedsAttention,
"executor lost",
nil,
)
require.NoError(t, err)
require.True(t, ok)
require.True(t, stopped)
}
func TestAutomationCancellationWinsApprovalRace(t *testing.T) {
f := newAutomationFixture(t)
a := f.create(t, types.AutomationPatch{})
f.suspendOnApproval(t, "op:x")
queued, err := f.svc.RunNow(f.ctx, a.ID)
require.NoError(t, err)
r := f.settleRun(t, queued.ID)
require.Equal(t, types.AutomationRunAwaitingApproval, r.Status)
// A remote cancellation can land after execute checked its context but
// before it records the approval suspension. Do not resume that run.
require.NoError(t, f.db.Model(r).Update("cancel_requested", true).Error)
ok, err := f.broker.Store().Transition(f.ctx, r.PendingPromptID, types.AgentPromptTransition{
From: []string{types.AgentPromptSuspended},
To: types.AgentPromptAnswered,
Response: types.JSON(`{"approved":true}`),
})
require.NoError(t, err)
require.True(t, ok)
require.NoError(t, f.svc.queueResume(f.ctx, r.ID, r.PendingPromptID))
require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now))
require.NoError(t, f.db.First(&r, "id = ?", r.ID).Error)
require.Equal(t, types.AutomationRunCancelled, r.Status)
}

View File

@ -36,6 +36,7 @@ type runOutcome struct {
answer string
usage *types.TokenUsage
promptID string
elapsed time.Duration
// pause, when set, pauses the automation with this reason.
pause string
}
@ -56,6 +57,7 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio
}()
go s.heartbeat(ctx, run.ID, cancel)
started := time.Now()
out := func() (out runOutcome) {
defer func() {
if r := recover(); r != nil {
@ -64,11 +66,15 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio
}()
return s.runOnce(ctx, run)
}()
if ctx.Err() != nil && out.status != types.AutomationRunAwaitingApproval {
out.elapsed = time.Since(started)
if ctx.Err() != nil {
var row types.AutomationRun
if err := s.db.WithContext(context.WithoutCancel(ctx)).Select("cancel_requested").
Where("id = ?", run.ID).Take(&row).Error; err == nil && row.CancelRequested {
out.status, out.reason = types.AutomationRunCancelled, "cancelled by the owner"
if out.promptID != "" {
s.closePrompt(context.WithoutCancel(ctx), out.promptID, types.AgentPromptCanceled, out.reason)
}
}
}
s.record(ctx, run, out)
@ -109,7 +115,10 @@ func (s *AutomationService) heartbeat(ctx context.Context, runID string, cancel
func (s *AutomationService) record(ctx context.Context, run *types.AutomationRun, out runOutcome) {
ctx = context.WithoutCancel(ctx)
run.Answer = clipRunes(out.answer, maxStoredAnswerRunes)
updates := map[string]any{"answer": run.Answer, "updated_at": s.now()}
updates := map[string]any{
"answer": run.Answer, "updated_at": s.now(),
"execution_duration_ms": gorm.Expr("execution_duration_ms + ?", out.elapsed.Milliseconds()),
}
if out.usage != nil {
if data, err := json.Marshal(out.usage); err == nil {
updates["usage"] = types.JSON(data)
@ -204,6 +213,7 @@ func (s *AutomationService) ownerContext(ctx context.Context, a *types.Automatio
// runOnce executes run, or continues it from its answered approval.
func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRun) runOutcome {
started := time.Now()
if s.sessions == nil {
return failed("the agent executor is unavailable")
}
@ -234,7 +244,14 @@ func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRu
if m := a.Limits.MaxDurationMinutes; m > 0 {
limit = time.Duration(m) * time.Minute
}
ctx, cancel := context.WithTimeout(ctx, limit)
remaining := limit - time.Duration(run.ExecutionDurationMS)*time.Millisecond - time.Since(started)
if remaining <= 0 {
if run.Resume && run.PendingPromptID != "" {
s.closePrompt(ctx, run.PendingPromptID, types.AgentPromptFailed, "execution time limit exceeded")
}
return failed(fmt.Sprintf("the run took longer than %s", limit))
}
ctx, cancel := context.WithTimeout(ctx, remaining)
defer cancel()
var resume *types.AgentResume
@ -524,7 +541,10 @@ func (s *AutomationService) agentQA(
}
mu.Lock()
defer mu.Unlock()
answer.IsCompleted = suspended == nil
answer.IsCompleted = suspended == nil || err != nil
if answer.IsCompleted {
answer.RunState = ""
}
answer.UpdatedAt = s.now()
if err != nil && answer.Content == "" {
answer.Content = err.Error()

View File

@ -163,6 +163,9 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro
}
var queued []*types.AutomationRun
if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunQueued).
Where(`NOT EXISTS (SELECT 1 FROM automation_runs AS holding
WHERE holding.automation_id = automation_runs.automation_id AND holding.status IN ? )`,
[]string{types.AutomationRunRunning, types.AutomationRunAwaitingApproval}).
Order("resume DESC, scheduled_at").Limit(free * 4).Find(&queued).Error; err != nil {
return err
}
@ -186,16 +189,6 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro
}
func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationRun, now time.Time) (bool, error) {
var holding int64
if err := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{
types.AutomationRunRunning, types.AutomationRunAwaitingApproval,
}).Count(&holding).Error; err != nil {
return false, err
}
if holding > 0 {
return false, nil
}
lease := now.Add(s.cfg.Lease)
updates := map[string]any{
"status": types.AutomationRunRunning, "owner": s.instance, "lease_until": lease, "updated_at": now,
@ -203,10 +196,33 @@ func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationR
if run.StartedAt == nil {
updates["started_at"] = now
}
res := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates)
if res.Error != nil || res.RowsAffected == 0 {
return false, res.Error
started := false
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
// Take the automation's write lock before reading its runs. A run-row
// CAS alone cannot serialize two different queued runs. Writing first
// also avoids a deferred read-to-write transaction upgrade on SQLite.
locked := tx.Model(&types.Automation{}).Where("id = ?", run.AutomationID).
UpdateColumn("version", gorm.Expr("version + 1"))
if locked.Error != nil || locked.RowsAffected == 0 {
return locked.Error
}
var holding int64
if err := tx.Model(&types.AutomationRun{}).
Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{
types.AutomationRunRunning, types.AutomationRunAwaitingApproval,
}).Count(&holding).Error; err != nil {
return err
}
if holding > 0 {
return nil
}
res := tx.Model(&types.AutomationRun{}).
Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates)
started = res.Error == nil && res.RowsAffected > 0
return res.Error
})
if err != nil || !started {
return false, err
}
run.Status, run.Owner, run.LeaseUntil = types.AutomationRunRunning, s.instance, &lease
if run.StartedAt == nil {
@ -266,20 +282,31 @@ func (s *AutomationService) reconcileAwaiting(ctx context.Context, now time.Time
if s.broker == nil {
return nil
}
var waiting []*types.AutomationRun
if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunAwaitingApproval).
Order("updated_at").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil {
return err
}
for _, run := range waiting {
if err := s.reconcileRun(ctx, run, now); err != nil {
logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err)
// Keyset pagination visits later approvals even when earlier ones stay
// open. Offsets would skip rows as reconciled runs leave the waiting set.
for cursor := ""; ; {
var waiting []*types.AutomationRun
if err := s.db.WithContext(ctx).
Where("status = ? AND id > ?", types.AutomationRunAwaitingApproval, cursor).
Order("id").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil {
return err
}
for _, run := range waiting {
if err := s.reconcileRun(ctx, run, now); err != nil {
logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err)
}
}
if len(waiting) < automationClaimBatch*2 {
return nil
}
cursor = waiting[len(waiting)-1].ID
}
return nil
}
func (s *AutomationService) reconcileRun(ctx context.Context, run *types.AutomationRun, now time.Time) error {
if run.CancelRequested {
return s.cancelRun(ctx, run, "cancelled by the owner")
}
waiting := []string{types.AutomationRunAwaitingApproval}
prompt, err := s.broker.Store().Get(ctx, run.PendingPromptID)
if err != nil {
@ -324,7 +351,8 @@ func (s *AutomationService) reconcileRun(ctx context.Context, run *types.Automat
// queue, to continue from the answer.
func (s *AutomationService) queueResume(ctx context.Context, runID, promptID string) error {
res := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
Where("id = ? AND status = ? AND pending_prompt_id = ?", runID, types.AutomationRunAwaitingApproval, promptID).
Where("id = ? AND status = ? AND pending_prompt_id = ? AND cancel_requested = ?",
runID, types.AutomationRunAwaitingApproval, promptID, false).
Updates(map[string]any{"status": types.AutomationRunQueued, "resume": true, "updated_at": s.now()})
if res.Error == nil && res.RowsAffected > 0 {
s.Wake()
@ -407,8 +435,15 @@ func (s *AutomationService) finishRun(
return false, res.Error
}
run.Status, run.Error, run.FinishedAt = status, reason, &now
// Only the winning finalizer may stop work. A stale approval reaper must
// not cancel a run that another scheduler has already resumed.
var stopErr error
if status == types.AutomationRunFailed || status == types.AutomationRunCancelled ||
status == types.AutomationRunNeedsAttention {
stopErr = s.stopRunDelegation(context.WithoutCancel(ctx), run)
}
s.settle(ctx, run)
return true, nil
return true, stopErr
}
// settle records a finished run on its automation, pauses it after too many

View File

@ -431,6 +431,7 @@ func (s *sessionService) AgentQA(
// run elsewhere. Session, attachments and images are reloaded from storage.
func resumableAgentRequest(ctx context.Context, req *types.QARequest) types.ResumableAgentRequest {
r := types.ResumableAgentRequest{
ActingVia: types.ActingViaFromContext(ctx),
Query: req.Query,
SummaryModelID: req.SummaryModelID,
ReasoningEffort: req.ReasoningEffort,

View File

@ -655,6 +655,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
}
automations.Bind(sessionService, broker, runs, streams)
automations.Events = sessions.CollaborationEvents
automations.StopDelegation = collaboration.StopOwned
broker.SetRunHooks(runs.Hooks())
runs.StopDelegation = func(ctx context.Context, p *types.AgentPrompt) error {
return collaboration.StopPrompt(ctx, p)

View File

@ -0,0 +1,22 @@
package database
import (
"path/filepath"
"testing"
"github.com/golang-migrate/migrate/v4/source"
"github.com/stretchr/testify/require"
)
// Check both migration registries without requiring a live PostgreSQL server.
// Independently merged migrations must not reuse a version in either dialect.
func TestMigrationSourcesHaveUniqueVersions(t *testing.T) {
root := sqliteRepoRoot(t)
for _, dialect := range []string{"sqlite", "versioned"} {
t.Run(dialect, func(t *testing.T) {
migrations, err := source.Open("file://" + filepath.Join(root, "migrations", dialect))
require.NoError(t, err)
require.NoError(t, migrations.Close())
})
}
}

View File

@ -0,0 +1,36 @@
package database
import (
"fmt"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestSQLiteAutomationDurationUpgradePreservesExistingRuns(t *testing.T) {
repoRoot := sqliteRepoRoot(t)
for _, version := range []int{45, 47} {
t.Run(fmt.Sprintf("from-v%d", version), func(t *testing.T) {
chdirAndRestore(t, copySQLiteMigrationsThrough(t, repoRoot, version))
dbPath := filepath.Join(t.TempDir(), "automations.db")
require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))
db := openSQLiteDB(t, dbPath)
_, err := db.Exec(`INSERT INTO automation_runs
(id, automation_id, tenant_id, scheduled_at, trigger_kind, status, answer, created_at, updated_at)
VALUES ('existing-run', 'automation', 1, CURRENT_TIMESTAMP, 'schedule',
'awaiting_approval', 'saved answer', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
require.NoError(t, err)
chdirAndRestore(t, repoRoot)
require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))
var duration int64
var answer, status string
require.NoError(t, db.QueryRow(`SELECT execution_duration_ms, answer, status
FROM automation_runs WHERE id = 'existing-run'`).Scan(&duration, &answer, &status))
require.Zero(t, duration)
require.Equal(t, "saved answer", answer)
require.Equal(t, "awaiting_approval", status)
})
}
}

View File

@ -52,6 +52,7 @@ var versionedSQLiteTables = []string{
// versionedSQLiteColumns maps each existing table to the columns that the
// versioned migrations add and the SQLite baseline was missing.
var versionedSQLiteColumns = map[string][]string{
"automation_runs": {"execution_duration_ms"}, // 000048
"model_catalog_configs": {"version", "overlay", "history", "updated_by", "updated_at"}, // 000031
"memory_subjects": {"extraction_state"}, // 000094
"memory_items": {"replaces_id"}, // 000094
@ -86,7 +87,7 @@ var versionedSQLiteColumns = map[string][]string{
}, // 000028
}
const expectedSQLiteMigrationVersion = 47
const expectedSQLiteMigrationVersion = 48
func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {
repoRoot := sqliteRepoRoot(t)
@ -262,19 +263,20 @@ func TestSQLiteSandboxAllocationMigrationRollsBack(t *testing.T) {
require.NoError(t, err)
t.Cleanup(func() { _, _ = m.Close() })
require.NoError(t, m.Steps(-1))
// Target the allocation migration even when newer migrations exist.
require.NoError(t, m.Migrate(46))
version, dirty = sqliteMigrationState(t, sqlDB)
require.Equal(t, expectedSQLiteMigrationVersion-1, version)
require.Equal(t, 46, version)
require.False(t, dirty)
for _, table := range []string{"session_sandbox_assignments", "sandbox_user_machines"} {
require.Falsef(t, sqliteTableExists(t, sqlDB, table), "down must drop %s", table)
}
require.True(t, sqliteColumnExists(t, sqlDB, "sessions", "owner_auth_source"),
"rolling back the allocation tables keeps the owner source of 000034")
"rolling back the allocation tables keeps the owner source of 000046")
require.NoError(t, m.Steps(1))
require.NoError(t, m.Migrate(47))
version, dirty = sqliteMigrationState(t, sqlDB)
require.Equal(t, expectedSQLiteMigrationVersion, version)
require.Equal(t, 47, version)
require.False(t, dirty)
require.True(t, sqliteTableExists(t, sqlDB, "session_sandbox_assignments"))
require.True(t, sqliteTableExists(t, sqlDB, "sandbox_user_machines"))

View File

@ -168,8 +168,10 @@ func (p *AgentPrompt) TimelineView() *PendingPromptView {
// ResumableAgentRequest is the part of a QARequest needed to rebuild the run
// on another replica. It holds the original query and durable request settings.
type ResumableAgentRequest struct {
AgentID string `json:"agent_id"`
Query string `json:"query,omitempty"`
// ActingVia keeps scheduler-owned runs out of ordinary chat recovery.
ActingVia string `json:"acting_via,omitempty"`
AgentID string `json:"agent_id"`
Query string `json:"query,omitempty"`
// AgentTenantID is the agent owner's workspace; it differs from the
// session's when the agent is shared.
AgentTenantID uint64 `json:"agent_tenant_id,omitempty"`

View File

@ -170,6 +170,9 @@ type AutomationRun struct {
Resume bool `json:"-" gorm:"not null;default:false"`
// CancelRequested stops the run at its next check.
CancelRequested bool `json:"cancel_requested" gorm:"not null;default:false"`
// ExecutionDurationMS accumulates execution segments across approvals;
// queued time and time spent waiting for an answer do not consume it.
ExecutionDurationMS int64 `json:"-" gorm:"not null;default:0"`
// Answer is the start of the final answer, for listings.
Answer string `json:"answer" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"`
Error string `json:"error" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"`

View File

@ -0,0 +1 @@
ALTER TABLE automation_runs DROP COLUMN execution_duration_ms;

View File

@ -0,0 +1 @@
ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;

View File

@ -0,0 +1 @@
ALTER TABLE automation_runs DROP COLUMN execution_duration_ms;

View File

@ -0,0 +1 @@
ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;